Orion setup improvements (#288)

* fix: deadlock issue caused by migration scripts

* gitignore src/types/

* update nodejs version to node:18

* added 'generate-migrations' makefile  command to generate migrations

* add npm command to get graphql chema

* add opentelemetry tracing integration with graphql-server and auth-server

* update docker.yml github workflow

* fix: DistributionBucketsCache init bug

* Improved offchain import/export script

* fix: dockerfile

* fix: dockerfile

* fix eslint issues

* fix checks.yml github workflow

* regenerate db migration files

* fix: Don't expose db container ports

* fix notification errors

* merge upstream/master

.env

@@ -72,3 +72,11 @@ EMAIL_NOTIFICATION_DELIVERY_MAX_ATTEMPTS=5
 APP_ASSET_STORAGE=https://raw.githubusercontent.com/Joystream/atlas-notification-assets/main/logos/gleev
 APP_NAME_ALT=Gleev.xyz
 NOTIFICATION_ASSET_ROOT=https://raw.githubusercontent.com/Joystream/atlas-notification-assets/main/icons
+
+# =====================================================================================
+# Telemetry
+# =====================================================================================
+
+# yes/no
+TELEMETRY_ENABLED=no
+TELEMETRY_ENDPOINT=http://apm-server:8200

.github/workflows/docker.yml

@@ -1,22 +1,58 @@
-name: Build and publish to Docker Hub
+name: Build & Publish Orion Docker image
 
 on:
-  push:
-    branches:
-      - 'master'
+  workflow_dispatch:
 
 jobs:
-  docker:
+  build_and_publish:
+    name: Build and Publish
     runs-on: ubuntu-latest
     steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18.x'
+
+      - name: Extract Package Version
+        id: extract_version
+        shell: bash
+        run: |
+          echo "orion_version=$(cat package.json | jq -r '.version')" >> $GITHUB_OUTPUT
+
+      - name: Make some space
+        shell: bash
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf "/usr/local/share/boost"
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        if: github.event_name == 'workflow_dispatch'
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build and push
-        id: docker_build
-        uses: docker/build-push-action@v2
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Build orion
+        uses: docker/build-push-action@v3
         with:
-          push: true
+          context: .
+          file: Dockerfile
+          push: false
+          load: true
           tags: joystream/orion:latest
+
+      - name: Push Orion Image
+        run: |
+          docker image tag joystream/orion:latest joystream/orion:${{ steps.extract_version.outputs.orion_version }}
+          docker push joystream/orion:${{ steps.extract_version.outputs.orion_version }}
+
+      - name: Push Orion latest tag
+        if: github.ref == 'refs/heads/master'
+        run: |
+          docker push joystream/orion:latest

.gitignore

@@ -8,6 +8,7 @@
 /.vscode
 /.idea
 src/model/generated
+src/types/
 src/auth-server/emails/templates/*.mst
 src/auth-server/emails/templates/preview
 /schema.graphql

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:16-alpine AS node
+FROM node:18-alpine AS node
 
 FROM node AS node-with-gyp
 RUN apk add g++ make python3
@@ -10,10 +10,13 @@ ADD package-lock.json .
 ADD assets assets
 RUN npm ci
 ADD tsconfig.json .
+ADD typegen.json .
+ADD joystream.jsonl .
 ADD src src
 ADD schema schema
 ADD scripts scripts
 RUN npx squid-typeorm-codegen
+RUN npx squid-substrate-typegen typegen.json
 RUN npm run build
 
 FROM node-with-gyp AS deps
@@ -45,4 +48,4 @@ CMD ["npm", "run", "processor-start"]
 
 
 FROM squid AS query-node
-CMD ["npm", "run", "query-node-start"]
+CMD ["npm", "run", "graphql-server-start"]

Makefile

@@ -12,7 +12,7 @@ build-docker:
 	@docker build . -t joystream/orion
 
 serve:
-	@npx squid-graphql-server --subscriptions
+	@npm run graphql-server-start
 
 serve-auth-api:
 	@npm run auth-server-start
@@ -23,6 +23,18 @@ migrate:
 dbgen:
 	@npx squid-typeorm-migration generate
 
+generate-migrations: 
+	@rm db/migrations/*-Data.js || true
+	@docker run -d --name temp_migrations_db \
+		-e POSTGRES_DB=squid \
+		-e POSTGRES_HOST_AUTH_METHOD=trust \
+		-v temp_migrations_db_volume:/var/lib/postgresql/data \
+		-v ./db/postgres.conf:/etc/postgresql/postgresql.conf \
+		-p 5555:5555 postgres:14 postgres -p 5555 || true
+	@export DB_PORT=5555 && sleep 5 && npx squid-typeorm-migration generate
+	@docker rm temp_migrations_db -vf || true
+	@docker volume rm temp_migrations_db_volume || true
+
 codegen:
 	@npm run generate:schema || true
 	@npx squid-typeorm-codegen
@@ -31,7 +43,7 @@ codegen:
 typegen:
 	@npx squid-substrate-typegen typegen.json
 
-prepare: install codegen build
+prepare: install typegen codegen build
 
 up-squid:
 	@docker network create joystream_default || true

db/migrations/2200000000000-Operator.js

@@ -21,11 +21,11 @@ module.exports = class Operator2300000000000 {
     `)
 
     // Create pg_stat_statements extension for analyzing query stats
-    await db.query(`CREATE EXTENSION pg_stat_statements`)
+    await db.query(`CREATE EXTENSION pg_stat_statements;`)
   }
 
   async down(db) {
     await db.query(`DELETE FROM "admin"."user" WHERE "is_root" = true;`)
-    await db.query(`DROP EXTENSION pg_stat_statements`)
+    await db.query(`DROP EXTENSION pg_stat_statements;`)
   }
 }

docker-compose.yml

@@ -10,7 +10,8 @@ services:
       POSTGRES_DB: squid
       POSTGRES_PASSWORD: squid
     ports:
-      - '${DB_PORT}:${DB_PORT}'
+      - '127.0.0.1:${DB_PORT}:${DB_PORT}'
+      - '[::1]:${DB_PORT}:${DB_PORT}'
     command: ['postgres', '-c', 'config_file=/etc/postgresql/postgresql.conf', '-p', '${DB_PORT}']
     shm_size: 1g
     volumes:
@@ -20,7 +21,7 @@ services:
   orion_processor:
     container_name: orion_processor
     hostname: orion_processor
-    image: node:14
+    image: node:18
     restart: unless-stopped
     env_file:
       - .env
@@ -40,13 +41,14 @@ services:
   orion_graphql-server:
     container_name: orion_graphql-server
     hostname: orion_graphql-server
-    image: node:14
+    image: node:18
     restart: unless-stopped
     env_file:
       - .env
       - docker.env
     environment:
       - SQD_TRACE=authentication
+      - OTEL_EXPORTER_OTLP_ENDPOINT=${TELEMETRY_ENDPOINT}
     depends_on:
       - orion_db
     volumes:
@@ -61,13 +63,14 @@ services:
   orion_auth-api:
     container_name: orion_auth-api
     hostname: orion_auth-api
-    image: node:14
+    image: node:18
     restart: unless-stopped
     env_file:
       - .env
       - docker.env
     environment:
       - SQD_TRACE=authentication
+      - OTEL_EXPORTER_OTLP_ENDPOINT=${TELEMETRY_ENDPOINT}
     depends_on:
       - orion_db
     volumes:

docs/developer-guide/commands.md

@@ -28,7 +28,7 @@
 - `npm run checks` - runs basic checks (linting, formatting, type checking), see: [Code style](./code-style.md)
 - `npm run db:migrate` - runs database migrations, same as `make migrate`
 - `npm run processor-start` - similar to `make process`, but doesn't run migrations
-- `npm run query-node-start` - similar to `make serve`
+- `npm run graphql-server-start` - similar to `make serve`
 - `npm run auth-server-start` - same as `make serve-auth-api`
 - `npm run tests:auth-api` - runs unit tests for the `auth-api` service (see: [Authentication API](./tutorials/authentication-api.md)).
 - `npm run offchain-state:export` - performs the offchain state export, see: [Preserving offchain state](./tutorials/preserving-offchain-state.md)

docs/operator-guide/examples/docker-compose.yml

@@ -48,7 +48,7 @@ services:
     environment:
       - DB_HOST=orion-db
       - GQL_PORT=${GRAPHQL_API_PORT}
-    command: ['npm', 'run', 'query-node-start']
+    command: ['npm', 'run', 'graphql-server-start']
     ports:
       - '127.0.0.1:${GRAPHQL_API_PORT}:${GRAPHQL_API_PORT}'
   # Orion Authentication API

entrypoints/auth-server.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -e
+
+# docker entrypoint fot auth api, to allow running with telemetry
+if [[ "$TELEMETRY_ENABLED" = "yes" ]]; then
+    node --require ./opentelemetry/index.js ./lib/auth-server/index.js $*
+else
+    node ./lib/auth-server/index.js $*
+fi

entrypoints/graphql-server.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -e
+
+# docker entrypoint fot graphql-server, to allow running with telemetry
+if [[ "$TELEMETRY_ENABLED" = "yes" ]]; then
+    node --require ./opentelemetry/index.js ./node_modules/@subsquid/graphql-server/bin/run.js --subscriptions $*
+else
+    node ./node_modules/@subsquid/graphql-server/bin/run.js --subscriptions $*
+fi

opentelemetry/.env

@@ -0,0 +1,8 @@
+# Required env variables for the Elasticsearch exporters
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:8200
+OTEL_RESOURCE_ATTRIBUTES="service.name=orion-graphql-server,deployment.environment=production"
+OTEL_METRICS_EXPORTER="otlp"
+
+# Optional env vars to configure the opentelemetry exporters
+OTEL_MAX_QUEUE_SIZE=8192        # 4 times of default queue size
+OTEL_MAX_EXPORT_BATCH_SIZE=1024 # 2 times of default batch size

opentelemetry/index.js

@@ -0,0 +1,59 @@
+const { DiagConsoleLogger, DiagLogLevel, diag } = require('@opentelemetry/api')
+const { getNodeAutoInstrumentations } = require('@opentelemetry/auto-instrumentations-node')
+
+/**
+ * Error: "@opentelemetry/instrumentation-grpc Module @grpc/grpc-js has
+ * been loaded before @opentelemetry/instrumentation-grpc so it might not work,
+ * please initialize it before requiring @grpc/grpc-js"
+ *
+ * Fix: "call getNodeAutoInstrumentations() before require('@opentelemetry/sdk-node');"
+ */
+//
+
+// Disable DNS instrumentation, because the instrumentation does not correctly patches `dns.lookup` function
+// if the function is converted to a promise-based method using `utils.promisify(dns.lookup)`
+// See: https://github.com/Joystream/joystream/pull/4779#discussion_r1262515887
+const instrumentations = getNodeAutoInstrumentations({
+  '@opentelemetry/instrumentation-dns': { enabled: false },
+  '@opentelemetry/instrumentation-pg': { enhancedDatabaseReporting: true },
+})
+
+const { NodeSDK } = require('@opentelemetry/sdk-node')
+const { OTLPMetricExporter } = require('@opentelemetry/exporter-metrics-otlp-proto')
+const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-proto')
+const { PeriodicExportingMetricReader } = require('@opentelemetry/sdk-metrics')
+const { BatchSpanProcessor } = require('@opentelemetry/sdk-trace-node')
+
+const path = require('path')
+require('dotenv').config({ path: path.resolve(__dirname, `.env`) })
+
+// For troubleshooting, set the log level to DiagLogLevel.DEBUG
+diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.INFO)
+
+function addInstrumentation() {
+  const instrumentation = new NodeSDK({
+    spanProcessor: new BatchSpanProcessor(new OTLPTraceExporter(), {
+      maxQueueSize: parseInt(process.env.OTEL_MAX_QUEUE_SIZE || '8192'),
+      maxExportBatchSize: parseInt(process.env.OTEL_MAX_EXPORT_BATCH_SIZE || '1024'),
+    }),
+    metricReader: new PeriodicExportingMetricReader({
+      exporter: new OTLPMetricExporter(),
+    }),
+    instrumentations,
+  })
+
+  // Start Opentelemetry NodeJS Instrumentation
+  diag.info('Starting tracing...')
+  instrumentation.start()
+
+  // gracefully shut down the SDK on process exit
+  process.on('SIGTERM', () => {
+    instrumentation
+      .shutdown()
+      .then(() => console.log('Tracing terminated'))
+      .catch((error) => console.log('Error terminating tracing', error))
+      .finally(() => process.exit(0))
+  })
+}
+
+addInstrumentation()