Merge pull request #321 from Joystream/dev

Release: `v3.4.0`

.gitignore

@@ -43,7 +43,6 @@ error.log
 Thumbs.db
 **/node_modules/
 
-# Serverless
-.serverless
-.webpack
+# Env files
 .env
+!opentelemetry/.env

CHANGELOG.md

@@ -1,3 +1,12 @@
+### 3.4.0
+
+- Enable opentelemetry integration for tracing
+- Support running httpApi and sync as separate services
+- **FIX**: Added timeout in `ContentMetadataService` to avoid infinite waiting for the video to be processed
+- **FIX**: Restarting chisel client container from inside yt-synch container
+- **FIX**: Skip uploading object to storage node if it already exists
+- **FIX**: `ytdlpClient.getVideos` method
+
 ### 3.3.0
 
 - Added `chisel` proxy setup with automated IP rotation mechanism to circumvent youtube IP blockage, also integrated proxy setup with yt-synch server. See [docs](socks5-proxy/SETUP.md) for more details

Dockerfile

@@ -4,6 +4,19 @@ FROM node:18
 # Set the working directory to /youtube-synch
 WORKDIR /youtube-synch
 
+# Install AWS CLI
+RUN apt-get update && \
+    apt-get install -y awscli && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install Docker manually
+RUN curl -fsSL https://get.docker.com -o get-docker.sh && \
+    sh get-docker.sh && \
+    rm get-docker.sh
+
+# Install node-gyp
+RUN npm install -g node-gyp
+
 # Copy the package.json and yarn.lock (or package-lock.json for npm) files
 COPY package.json yarn.lock ./
 
@@ -17,4 +30,4 @@ COPY . .
 RUN yarn build
 
 # Set the command to run when a container based on the image is started
-CMD ["./bin/run", "start"]
+CMD ["./scripts/start-youtube-synch-httpApi.sh"]

docker-compose.elasticsearch.yml

@@ -6,10 +6,11 @@ services:
     image: docker.elastic.co/elasticsearch/elasticsearch:8.7.0
     container_name: elasticsearch
     environment:
+      # Ref: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/security-minimal-setup.html#_enable_elasticsearch_security_features
+      - xpack.security.enabled=true
       - discovery.type=single-node
       - bootstrap.memory_lock=true
       - 'ES_JAVA_OPTS=-Xms512m -Xmx512m'
-      - xpack.security.enabled=true
       - ELASTIC_PASSWORD=${ELASTIC_PASSWORD:-password}
     ulimits:
       memlock:
@@ -19,29 +20,45 @@ services:
       - es-data:/usr/share/elasticsearch/data
     ports:
       - 127.0.0.1:9200:9200
+    networks:
+      - youtube-synch
+
   # Ref: https://www.elastic.co/guide/en/kibana/8.7/docker.html
   kibana:
     image: docker.elastic.co/kibana/kibana:8.7.0
     container_name: kibana
-    ports:
-      - 127.0.0.1:5601:5601
     depends_on:
       - elasticsearch
     environment:
       ELASTICSEARCH_HOSTS: http://elasticsearch:9200
       ELASTICSEARCH_SERVICEACCOUNTTOKEN: ${ELASTICSEARCH_SERVICEACCOUNTTOKEN}
       # Ref: https://www.elastic.co/guide/en/kibana/current/xpack-security-secure-saved-objects.html#xpack-security-secure-saved-objects
       XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: ${XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY}
+    ports:
+      - 127.0.0.1:5601:5601
+    networks:
+      - youtube-synch
+
+  # Ref: https://www.elastic.co/guide/en/apm/guide/8.7/running-on-docker.html
+  apm-server:
+    image: docker.elastic.co/apm/apm-server:8.7.0
+    depends_on:
+      - elasticsearch
+    command: |
+      --strict.perms=false -e 
+      -E output.elasticsearch.hosts=["elasticsearch:9200"] 
+      -E output.elasticsearch.username=${ELASTIC_USERNAME:-elastic} 
+      -E output.elasticsearch.password=${ELASTIC_PASSWORD:-password}
+    ports:
+      - 8200:8200
+    networks:
+      - youtube-synch
 
 volumes:
   es-data:
     driver: local
-  cache:
-    driver: local
-  data:
-    driver: local
 
+# Join youtube-synch network (from root docker-compose)
 networks:
-  joystream:
+  youtube-synch:
     external: true
-    name: youtube-synch_default

docker-compose.yml

@@ -1,8 +1,8 @@
 version: '3.4'
 services:
-  youtube-synch:
+  youtube-synch_httpApi:
     image: youtube-synch:${IMAGE_TAG}
-    container_name: youtube-synch
+    container_name: youtube-synch_httpApi
     restart: on-failure
     depends_on:
       - redis
@@ -11,15 +11,13 @@ services:
       dockerfile: Dockerfile
     volumes:
       - ./local/logs:/youtube-synch/local/logs
-      - ./local/data:/youtube-synch/local/data
       # mount Google Cloud's Application Default Credentials file. A default bind mount is created
-      # as workaround for scenario when `YT_SYNCH__YOUTUBE__ADC_KEY_FILE_PATH` will be undefined, 
+      # as workaround for scenario when `YT_SYNCH__YOUTUBE__ADC_KEY_FILE_PATH` will be undefined,
       # since docker-compose does not support creating bind-mount volume with empty path.
       - ${YT_SYNCH__YOUTUBE__ADC_KEY_FILE_PATH:-./package.json}:${YT_SYNCH__YOUTUBE__ADC_KEY_FILE_PATH:-/package.json}:ro
       # mount the AWS credentials file to docker container home directory
       - ~/.aws/:/root/.aws:ro
     env_file:
-      # relative to working directory where docker-compose was run from
       - .env
     environment:
       DEPLOYMENT_ENV: ${DEPLOYMENT_ENV}
@@ -32,7 +30,7 @@ services:
       # YT_SYNCH__HTTP_API__OWNER_KEY: ${YT_SYNCH__HTTP_API__OWNER_KEY}
       # YT_SYNCH__YOUTUBE__CLIENT_ID: ${YT_SYNCH__YOUTUBE__CLIENT_ID}
       # YT_SYNCH__YOUTUBE__CLIENT_SECRET: ${YT_SYNCH__YOUTUBE__CLIENT_SECRET}
-      # YT_SYNCH__AWS__ENDPOINT: ${YT_SYNCH__AWS__ENDPOINT}
+      # YT_SYNCH__AWS__ENDPOINT: http://host.docker.internal:4566
       # YT_SYNCH__AWS__CREDENTIALS__ACCESS_KEY_ID: ${YT_SYNCH__AWS__CREDENTIALS__ACCESS_KEY_ID}
       # YT_SYNCH__AWS__CREDENTIALS__SECRET_ACCESS_KEY: ${YT_SYNCH__AWS__CREDENTIALS__SECRET_ACCESS_KEY}
       # YT_SYNCH__JOYSTREAM__CHANNEL_COLLABORATOR__ACCOUNT: ${YT_SYNCH__JOYSTREAM__CHANNEL_COLLABORATOR__ACCOUNT}
@@ -41,14 +39,64 @@ services:
       # YT_SYNCH__JOYSTREAM__APP__ACCOUNT_SEED: ${YT_SYNCH__JOYSTREAM__APP__ACCOUNT_SEED}
       # YT_SYNCH__JOYSTREAM__FAUCET__ENDPOINT: ${YT_SYNCH__JOYSTREAM__FAUCET__ENDPOINT}
       # YT_SYNCH__JOYSTREAM__FAUCET__CAPTCHA_BYPASS_KEY: ${YT_SYNCH__JOYSTREAM__FAUCET__CAPTCHA_BYPASS_KEY}
+      OTEL_EXPORTER_OTLP_ENDPOINT: ${TELEMETRY_ENDPOINT}
     ports:
       - 127.0.0.1:${YT_SYNCH__HTTP_API__PORT}:${YT_SYNCH__HTTP_API__PORT}
     networks:
       - youtube-synch
+    command: ['./scripts/start-youtube-synch-httpApi.sh', '--service', 'httpApi']
+
+  youtube-synch_sync:
+    image: youtube-synch:${IMAGE_TAG}
+    container_name: youtube-synch_sync
+    restart: on-failure
+    depends_on:
+      - redis
+    build:
+      context: .
+      dockerfile: Dockerfile
+    volumes:
+      # Docker socket mounting is required to restart the socks5 proxy container from inside this container
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./local/logs:/youtube-synch/local/logs
+      - ./local/data:/youtube-synch/local/data
+      # mount Google Cloud's Application Default Credentials file. A default bind mount is created
+      # as workaround for scenario when `YT_SYNCH__YOUTUBE__ADC_KEY_FILE_PATH` will be undefined,
+      # since docker-compose does not support creating bind-mount volume with empty path.
+      - ${YT_SYNCH__YOUTUBE__ADC_KEY_FILE_PATH:-./package.json}:${YT_SYNCH__YOUTUBE__ADC_KEY_FILE_PATH:-/package.json}:ro
+      # mount the AWS credentials file to docker container home directory
+      - ~/.aws/:/root/.aws:ro
+    env_file:
+      # relative to working directory where docker-compose was run from
+      - .env
+    environment:
+      DEPLOYMENT_ENV: ${DEPLOYMENT_ENV}
+      # YT_SYNCH__LOGS__ELASTIC: "{\"level\":\"debug\",\"endpoint\":\"http://elasticsearch:9200\"}"
+      # YT_SYNCH__ENDPOINTS__QUERY_NODE: ${YT_SYNCH__ENDPOINTS__QUERY_NODE}
+      # YT_SYNCH__ENDPOINTS__JOYSTREAM_NODE_WS: ${YT_SYNCH__ENDPOINTS__JOYSTREAM_NODE_WS}
+      YT_SYNCH__ENDPOINTS__REDIS__HOST: redis
+      YT_SYNCH__ENDPOINTS__REDIS__PORT: ${YT_SYNCH__ENDPOINTS__REDIS__PORT}
+      # YT_SYNCH__HTTP_API__PORT: ${YT_SYNCH__HTTP_API__PORT}
+      # YT_SYNCH__HTTP_API__OWNER_KEY: ${YT_SYNCH__HTTP_API__OWNER_KEY}
+      # YT_SYNCH__YOUTUBE__CLIENT_ID: ${YT_SYNCH__YOUTUBE__CLIENT_ID}
+      # YT_SYNCH__YOUTUBE__CLIENT_SECRET: ${YT_SYNCH__YOUTUBE__CLIENT_SECRET}
+      # YT_SYNCH__AWS__ENDPOINT: http://host.docker.internal:4566
+      # YT_SYNCH__AWS__CREDENTIALS__ACCESS_KEY_ID: ${YT_SYNCH__AWS__CREDENTIALS__ACCESS_KEY_ID}
+      # YT_SYNCH__AWS__CREDENTIALS__SECRET_ACCESS_KEY: ${YT_SYNCH__AWS__CREDENTIALS__SECRET_ACCESS_KEY}
+      # YT_SYNCH__JOYSTREAM__CHANNEL_COLLABORATOR__ACCOUNT: ${YT_SYNCH__JOYSTREAM__CHANNEL_COLLABORATOR__ACCOUNT}
+      # YT_SYNCH__JOYSTREAM__CHANNEL_COLLABORATOR__MEMBER_ID: ${YT_SYNCH__JOYSTREAM__CHANNEL_COLLABORATOR__MEMBER_ID}
+      # YT_SYNCH__JOYSTREAM__APP__NAME: ${YT_SYNCH__JOYSTREAM__APP__NAME}
+      # YT_SYNCH__JOYSTREAM__APP__ACCOUNT_SEED: ${YT_SYNCH__JOYSTREAM__APP__ACCOUNT_SEED}
+      # YT_SYNCH__JOYSTREAM__FAUCET__ENDPOINT: ${YT_SYNCH__JOYSTREAM__FAUCET__ENDPOINT}
+      # YT_SYNCH__JOYSTREAM__FAUCET__CAPTCHA_BYPASS_KEY: ${YT_SYNCH__JOYSTREAM__FAUCET__CAPTCHA_BYPASS_KEY}
+    networks:
+      - youtube-synch
+    command: ['./bin/run', 'start', '--service', 'sync']
+
   redis:
     image: redis:7.2.1
     container_name: redis
-    command: [ "redis-server", "--maxmemory-policy", "noeviction" ]
+    command: ['redis-server', '--maxmemory-policy', 'noeviction']
     ports:
       - 127.0.0.1:${YT_SYNCH__ENDPOINTS__REDIS__PORT}:${YT_SYNCH__ENDPOINTS__REDIS__PORT}
     networks:

opentelemetry/.env

@@ -0,0 +1,8 @@
+# Required env variables for the Elasticsearch exporters
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:8200
+OTEL_RESOURCE_ATTRIBUTES="service.name=youtube-synch-httpApi,deployment.environment=production"
+OTEL_METRICS_EXPORTER="otlp"
+
+# Optional env vars to configure the opentelemetry exporters
+OTEL_MAX_QUEUE_SIZE=8192        # 4 times of default queue size
+OTEL_MAX_EXPORT_BATCH_SIZE=1024 # 2 times of default batch size

opentelemetry/index.js

@@ -0,0 +1,56 @@
+const { DiagConsoleLogger, DiagLogLevel, diag } = require('@opentelemetry/api')
+const { getNodeAutoInstrumentations } = require('@opentelemetry/auto-instrumentations-node')
+
+/**
+ * Error: "@opentelemetry/instrumentation-grpc Module @grpc/grpc-js has
+ * been loaded before @opentelemetry/instrumentation-grpc so it might not work,
+ * please initialize it before requiring @grpc/grpc-js"
+ *
+ * Fix: "call getNodeAutoInstrumentations() before require('@opentelemetry/sdk-node');"
+ */
+//
+
+// Disable DNS instrumentation, because the instrumentation does not correctly patches `dns.lookup` function
+// if the function is converted to a promise-based method using `utils.promisify(dns.lookup)`
+// See: https://github.com/Joystream/joystream/pull/4779#discussion_r1262515887
+const instrumentations = getNodeAutoInstrumentations({ '@opentelemetry/instrumentation-dns': { enabled: false } })
+
+const { NodeSDK } = require('@opentelemetry/sdk-node')
+const { OTLPMetricExporter } = require('@opentelemetry/exporter-metrics-otlp-proto')
+const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-proto')
+const { PeriodicExportingMetricReader } = require('@opentelemetry/sdk-metrics')
+const { BatchSpanProcessor } = require('@opentelemetry/sdk-trace-node')
+
+const path = require('path')
+require('dotenv').config({ path: path.resolve(__dirname, `.env`) })
+
+// For troubleshooting, set the log level to DiagLogLevel.DEBUG
+diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.INFO)
+
+function addInstrumentation() {
+  const instrumentation = new NodeSDK({
+    spanProcessor: new BatchSpanProcessor(new OTLPTraceExporter(), {
+      maxQueueSize: parseInt(process.env.OTEL_MAX_QUEUE_SIZE || '8192'),
+      maxExportBatchSize: parseInt(process.env.OTEL_MAX_EXPORT_BATCH_SIZE || '1024'),
+    }),
+    metricReader: new PeriodicExportingMetricReader({
+      exporter: new OTLPMetricExporter(),
+    }),
+    instrumentations,
+  })
+
+  // Start Opentelemetry NodeJS Instrumentation
+  diag.info('Starting tracing...')
+  instrumentation.start()
+
+  // gracefully shut down the SDK on process exit
+  process.on('SIGTERM', () => {
+    instrumentation
+      .shutdown()
+      .then(() => console.log('Tracing terminated'))
+      .catch((error) => console.log('Error terminating tracing', error))
+      .finally(() => process.exit(0))
+  })
+}
+
+addInstrumentation()

package.json

@@ -1,14 +1,14 @@
 {
   "name": "youtube-sync",
-  "version": "3.3.0",
+  "version": "3.4.0",
   "license": "MIT",
   "scripts": {
     "postpack": "rm -f oclif.manifest.json",
     "prepack": "rm -rf lib && tsc -b && oclif-dev manifest && oclif-dev readme",
     "build": "rm -rf lib && tsc --build tsconfig.json && cp -R ./src/infrastructure/pulumi ./lib/infrastructure/pulumi",
-    "start": "DEPLOYMENT_ENV=dev ./bin/run start",
+    "start": "DEPLOYMENT_ENV=dev ./scripts/start-youtube-synch-httpApi.sh",
     "lint": "eslint ./src --ext .ts",
-    "start:prod": "./bin/run start",
+    "start:prod": "./scripts/start-youtube-synch-httpApi.sh",
     "docker:start": "DEPLOYMENT_ENV=dev IMAGE_TAG=$(./scripts/code-shasum.sh) docker-compose up -d",
     "docker:start:prod": "IMAGE_TAG=$(./scripts/code-shasum.sh) docker-compose up -d",
     "dynamodb:local:start": "yarn dynamodb:local:stop && export DEPLOYMENT_ENV=local && ./src/infrastructure/deploy.sh",
@@ -68,6 +68,14 @@
     "typescript": "4.5.2"
   },
   "dependencies": {
+    "@opentelemetry/api": "^1.4.1",
+    "@opentelemetry/auto-instrumentations-node": "0.37.0",
+    "@opentelemetry/core": "1.13.0",
+    "@opentelemetry/exporter-metrics-otlp-http": "^0.39.1",
+    "@opentelemetry/exporter-metrics-otlp-proto": "^0.39.1",
+    "@opentelemetry/exporter-trace-otlp-http": "0.39.1",
+    "@opentelemetry/sdk-metrics": "1.13.0",
+    "@opentelemetry/sdk-node": "^0.39.1",
     "@apollo/client": "^3.2.5",
     "@bull-board/express": "^5.9.1",
     "@elastic/ecs-winston-format": "^1.3.1",

scripts/start-elasticsearch.sh

@@ -14,6 +14,9 @@ set +a
 ELASTIC_USERNAME=${ELASTIC_USERNAME:="elastic"}
 ELASTIC_PASSWORD=${ELASTIC_PASSWORD:="password"}
 
+# Remove elasticsearch stack containers & volumes
+docker-compose -f ../docker-compose.elasticsearch.yml down -v
+
 ## Run docker-compose to start elasticsearch container
 docker-compose -f ../docker-compose.elasticsearch.yml up -d elasticsearch
 
@@ -26,10 +29,31 @@ sleep 30
 export XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=$(openssl rand -base64 24)
 
 # Generate the service token
-response=$(curl -f -s -X POST -u "${ELASTIC_USERNAME}":"${ELASTIC_PASSWORD}" "http://localhost:9200/_security/service/elastic/kibana/credential/token/my_kibana_token" -H 'Content-Type: application/json')
+# Ref: https://www.elastic.co/guide/en/elasticsearch/reference/current/service-accounts.html#service-accounts-tokens
+response=$(curl -s -w "\n%{http_code}\n" -X POST -u "${ELASTIC_USERNAME}":"${ELASTIC_PASSWORD}" "http://localhost:9200/_security/service/elastic/kibana/credential/token/my_kibana_token" -H 'Content-Type: application/json')
+response_body=$(echo "$response" | head -n1)
+status_code=$(echo "$response" | tail -n1)
+
+if [ -z "$status_code" ]; then
+    echo "Error: Did not receive a status code from the server"
+    exit 1
+fi
+
+if [ "$status_code" -ne 200 ]; then
+    error_message=$(echo $response_body | jq -r '.error.root_cause')
+    echo -e "\nError: Failed to generate the service token: \n$error_message"
+    exit 1
+fi
 
 # Extract & export the token from the response
-export ELASTICSEARCH_SERVICEACCOUNTTOKEN=$(echo $response | jq -r '.token.value')
+export ELASTICSEARCH_SERVICEACCOUNTTOKEN=$(echo $response_body | jq -r '.token.value')
+
+echo 'Starting for Kibana...'
 
 ## Run docker-compose to start kibana container
 docker-compose -f ../docker-compose.elasticsearch.yml up -d kibana
+
+echo 'Starting APM Server...'
+
+## Run docker-compose to start apm-server container
+docker-compose -f ../docker-compose.elasticsearch.yml up -d apm-server

scripts/start-youtube-synch-httpApi.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+if [[ "$TELEMETRY_ENABLED" = "yes" ]]; then
+    node --require ./opentelemetry/index.js ./bin/run start $*
+else
+    echo "Starting YouTube Sync Service without telemetry..."
+    ./bin/run start $*
+fi

socks5-proxy/restart-ec2-proxy-instance.sh

@@ -17,8 +17,6 @@ set +a
 # Set your EC2 Instance ID and region
 INSTANCE_ID=$INSTANCE_ID
 
-# Fingerprint (Key) needed to connect to the Chisel Sever by Chisel Client running on the client machine
-
 # Stopping an instance
 echo -e "Stopping EC2 proxy server instance... \n"
 aws ec2 stop-instances --instance-ids $INSTANCE_ID

socks5-proxy/start-proxy-client.sh

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 set -e
 
 SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")"
@@ -24,7 +26,7 @@ export IP_ADDRESS=$IP_ADDRESS
 
 # Restart the docker-compose chisel client service
 echo -e "Restarting Chisel Client... \n"
-docker rm -f chisel-client >/dev/null 2>&1
+docker rm -f chisel-client || true
 
 export COMPOSE_HTTP_TIMEOUT=120 # Increase to 120 seconds
-docker-compose -f ./docker-compose.chisel.yml up -d chisel-client
+docker compose -f ./docker-compose.chisel.yml up -d chisel-client