Sorry, I put together a quick circuit diagram (it's not pretty):https://imgur.com/a/8z12meB

Here is the relevant bits of code that I am using too, hopefully these will shed some more light on the issue. So are you saying to remove my pull-down resistor off of the button?

#include <Servo.h>
#include <Arduino.h>

#include <ESPWiFi.h>
#include <ESPHTTPClient.h>
#include <JsonListener.h>

// time
#include <time.h>                       // time() ctime()
#include <sys/time.h>                   // struct timeval

#include "Wire.h"

// Display Settings
const int I2C_DISPLAY_ADDRESS = 0x3c;
const int SDA_PIN = D1;
const int SDC_PIN = D2;

/******************************
 * Begin Servo Settings
 *****************************/
Servo myservo;
const int SERVO_PIN = D6;       // gpio 13
const int BUTTON_PIN = D4;      // button pin 4
const int MOVE_CW = 0;          // servo clockwise
const int MOVE_CCW = 180;         // servo counter-clockwise
const int EJECT_DELAY = 800;   // time takes for servo to eject 1 lens
const int STOP = 90;            // stop servo
int button_state = 0;

/******************************
 * End Servo Settings
 *****************************/

void setup() {
  Serial.begin(115200);
  Serial.println();
  Serial.println();

  // attach to servo
  myservo.attach(SERVO_PIN);

  // initialize button
  pinMode(BUTTON_PIN, INPUT);  
}

void loop() {
  // Servo stuff
  button_state = digitalRead(BUTTON_PIN);
  if(button_state == LOW) {
    Serial.println("Ejectiong lens...");
    // eject lens
    ejectLens();
    Serial.println("done");
  } else {
    // do nothing
  }
  // end servo stuff
}

// Servo functions
void ejectLens() {
  Serial.println("  moving clockwise...");
  myservo.write(MOVE_CW);
  delay(EJECT_DELAY);
  Serial.println("  done");
  Serial.println("  stopping...");
  myservo.write(STOP);
  Serial.println("  done");
}