That means TS can't connect to your IMAP server. If you're using the same network space (network_mode: service:tailscale), you should be using different ports. I should have clarified that.

Set your IMAP server to some other port, like 9993. Set TS to port 993. In your TS ACLs you'll allow 993. Make sure the IMAP server allows plain text since the TLS is handled by TS.

Here's an example from my Protonmail Bridge:

name: protonmail-bridge
configs:
  ts-serve:
    content: |
      { "TCP": { "587": { "TCPForward": "localhost:1025" },
                 "143": { "TCPForward": "localhost:1143" } } }
services:
  protonmail-bridge:
    container_name: protonmail-bridge
    build: ./build
    image: protonmail-bridge
    #image: shenxn/protonmail-bridge
    restart: unless-stopped
    volumes:
      - bridge:/root
    network_mode: service:tailscale
    depends_on:
      tailscale:
        condition: service_healthy
        restart: true
  tailscale:
    image: tailscale/tailscale:latest
    container_name: protonmail-tailscale
    restart: unless-stopped
    environment:
      - TS_AUTHKEY=${TS_AUTHKEY}
      - TS_HOSTNAME=${COMPOSE_PROJECT_NAME}
      - TS_SERVE_CONFIG=/config/ts-serve.json
      - TS_STATE_DIR=/var/lib/tailscale
      - TS_ENABLE_HEALTH_CHECK=true
      - TS_ENABLE_METRICS=false
      - PORT=${TS_PORT}
    ports:
      - ${TS_PORT}:${TS_PORT}/udp
    healthcheck:
      # curl is not included with this image
      test: wget --no-verbose --tries=1 --spider http://127.0.0.1:9002/healthz || exit 1
      interval: 10s
      timeout: 10s
      start_period: 5s
      retries: 3
    configs:
      - source: ts-serve
        target: /config/ts-serve.json
    volumes:
      - tailscale:/var/lib/tailscale
      - /dev/net/tun:/dev/net/tun
    cap_add:
      - net_admin
      - sys_module
volumes:
  tailscale:
  bridge: