Merge pull request #323 from zeeshanakram3/replace-qn-with-orion

Replace QN with Orion for Graphql queries
Joystream · Feb 27, 2024 · dbc013b · dbc013b
2 parents 614c913 + a1fa8d2
commit dbc013b
Show file tree

Hide file tree

Showing 19 changed files with 21,909 additions and 35,232 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,9 @@
+### 3.6.0
+
+- Replace `Query-Node` with `Orion` for graphql queries, and remove all unused graphql query definitions.
+- **FIX**: use `GraphQLWsLink` instead of deprecated `WebSocketLink` for graphql subscription, as `WebSocketLink` based subscription does not work with Orion API.
+- **FIX**: Properly handle error when failing to download removed/deleted videos from Youtube.
+
 ### 3.5.0
 
 - Add support for setting `isShort` field in the video metadata (indicating whether video is a short format, vertical video or not) when creating the video.

diff --git a/docs/cli/start.md b/docs/cli/start.md
@@ -1,21 +1,23 @@
 `youtube-synch start`
 =====================
 
-Start the node
+Start the Youtube-Synch service/s.
 
 * [`youtube-synch start`](#youtube-synch-start)
 
 ## `youtube-synch start`
 
-Start the node
+Start the Youtube-Synch service/s.
 
 ```
 USAGE
   $ youtube-synch start
 
 OPTIONS
-  -c, --configPath=configPath  [default: ./config.yml] Path to config JSON/YAML file (relative to current working
-                               directory)
+  -c, --configPath=configPath    [default: ./config.yml] Path to config JSON/YAML file (relative to current working
+                                 directory)
 
-  -y, --yes                    Answer "yes" to any prompt, skipping any manual confirmations
+  -y, --yes                      Answer "yes" to any prompt, skipping any manual confirmations
+
+  --service=(httpApi|sync|both)  [default: both] Which service to start
 ```
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "youtube-sync",
-  "version": "3.5.0",
+  "version": "3.6.0",
   "license": "MIT",
   "scripts": {
     "postpack": "rm -f oclif.manifest.json",
@@ -68,15 +68,7 @@
     "typescript": "4.5.2"
   },
   "dependencies": {
-    "@opentelemetry/api": "^1.4.1",
-    "@opentelemetry/auto-instrumentations-node": "0.37.0",
-    "@opentelemetry/core": "1.13.0",
-    "@opentelemetry/exporter-metrics-otlp-http": "^0.39.1",
-    "@opentelemetry/exporter-metrics-otlp-proto": "^0.39.1",
-    "@opentelemetry/exporter-trace-otlp-http": "0.39.1",
-    "@opentelemetry/sdk-metrics": "1.13.0",
-    "@opentelemetry/sdk-node": "^0.39.1",
-    "@apollo/client": "^3.2.5",
+    "@apollo/client": "3.8.8",
     "@bull-board/express": "^5.9.1",
     "@elastic/ecs-winston-format": "^1.3.1",
     "@ffmpeg-installer/ffmpeg": "^1.1.0",
@@ -95,6 +87,14 @@
     "@oclif/command": "^1",
     "@oclif/config": "^1",
     "@oclif/plugin-help": "^3",
+    "@opentelemetry/api": "^1.4.1",
+    "@opentelemetry/auto-instrumentations-node": "0.37.0",
+    "@opentelemetry/core": "1.13.0",
+    "@opentelemetry/exporter-metrics-otlp-http": "^0.39.1",
+    "@opentelemetry/exporter-metrics-otlp-proto": "^0.39.1",
+    "@opentelemetry/exporter-trace-otlp-http": "0.39.1",
+    "@opentelemetry/sdk-metrics": "1.13.0",
+    "@opentelemetry/sdk-node": "^0.39.1",
     "@pulumi/aws": "^4.35.0",
     "@pulumi/aws-apigateway": "^0.0.5",
     "@pulumi/awsx": "^0.32.0",
@@ -123,7 +123,8 @@
     "express": "^4.17.1",
     "fluent-ffmpeg": "^2.1.2",
     "form-data": "^4.0.0",
-    "graphql": "^15.8.0",
+    "graphql": "^16.8.1",
+    "graphql-ws": "^5.15.0",
     "inquirer": "^8.1.2",
     "inquirer-date-prompt": "^2.0.0",
     "ioredis": "^5.3.2",

diff --git a/scripts/start-youtube-synch-httpApi.sh b/scripts/start-youtube-synch-httpApi.sh
@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 
 if [[ "$TELEMETRY_ENABLED" = "yes" ]]; then
+    echo "Starting YouTube Sync Service with telemetry enabled..."
     node --require ./opentelemetry/index.js ./bin/run start $*
 else
-    echo "Starting YouTube Sync Service without telemetry..."
     ./bin/run start $*
 fi
diff --git a/src/services/httpApi/controllers/channels.ts b/src/services/httpApi/controllers/channels.ts
@@ -106,7 +106,7 @@ export class ChannelsController {
       // reset authorization code to prevent repeated save channel requests by authorization code re-use
       const updatedUser: YtUser = { ...user, email, authorizationCode: randomBytes(10).toString('hex') }
 
-      const joystreamChannelLanguageIso = jsChannel.language?.iso
+      const joystreamChannelLanguageIso = jsChannel.language || undefined
 
       // If channel already exists in the DB (in `OptedOut` state), then we
       // associate most properties of existing channel record with the new

diff --git a/src/services/query-node/api.ts b/src/services/query-node/api.ts
@@ -10,11 +10,11 @@ import {
   split,
 } from '@apollo/client/core'
 import { onError } from '@apollo/client/link/error'
-import { WebSocketLink } from '@apollo/client/link/ws'
+import { GraphQLWsLink } from '@apollo/client/link/subscriptions'
 import { getMainDefinition } from '@apollo/client/utilities'
 import { MemberId, VideoId } from '@joystream/types/primitives'
-import BN from 'bn.js'
 import fetch from 'cross-fetch'
+import { createClient } from 'graphql-ws'
 import { Logger } from 'winston'
 import ws from 'ws'
 import { ExitCodes, QueryNodeApiError } from '../../types/errors'
@@ -23,38 +23,15 @@ import { StorageNodeInfo } from '../runtime/types'
 import {
   AppFieldsFragment,
   ChannelFieldsFragment,
-  DataObjectInfoFragment,
-  DistributionBucketFamilyFieldsFragment,
   GetAppsByName,
   GetAppsByNameQuery,
   GetAppsByNameQueryVariables,
   GetChannelById,
   GetChannelByIdQuery,
   GetChannelByIdQueryVariables,
-  GetDataObjectsByBagId,
-  GetDataObjectsByBagIdQuery,
-  GetDataObjectsByBagIdQueryVariables,
-  GetDataObjectsByChannelId,
-  GetDataObjectsByChannelIdQuery,
-  GetDataObjectsByChannelIdQueryVariables,
-  GetDataObjectsByVideoId,
-  GetDataObjectsByVideoIdQuery,
-  GetDataObjectsByVideoIdQueryVariables,
-  GetDistributionFamiliesAndBuckets,
-  GetDistributionFamiliesAndBucketsQuery,
-  GetDistributionFamiliesAndBucketsQueryVariables,
   GetMemberById,
   GetMemberByIdQuery,
   GetMemberByIdQueryVariables,
-  GetMembersByIds,
-  GetMembersByIdsQuery,
-  GetMembersByIdsQueryVariables,
-  GetStorageBagInfoForAsset,
-  GetStorageBagInfoForAssetQuery,
-  GetStorageBagInfoForAssetQueryVariables,
-  GetStorageBuckets,
-  GetStorageBucketsQuery,
-  GetStorageBucketsQueryVariables,
   GetStorageNodesInfoByBagId,
   GetStorageNodesInfoByBagIdQuery,
   GetStorageNodesInfoByBagIdQueryVariables,
@@ -70,10 +47,6 @@ import {
   QueryNodeStateFieldsFragment,
   QueryNodeStateSubscription,
   QueryNodeStateSubscriptionVariables,
-  StorageBucketsCount,
-  StorageBucketsCountQuery,
-  StorageBucketsCountQueryVariables,
-  StorageNodeInfoFragment,
   VideoFieldsFragment,
 } from './generated/queries'
 import { Maybe } from './generated/schema'
@@ -106,13 +79,12 @@ export class QueryNodeApi {
     })
 
     const queryLink = from([errorLink, new HttpLink({ uri: endpoint, fetch })])
-    const wsLink = new WebSocketLink({
-      uri: endpoint,
-      options: {
-        reconnect: true,
-      },
-      webSocketImpl: ws,
-    })
+    const wsLink = new GraphQLWsLink(
+      createClient({
+        url: endpoint,
+        webSocketImpl: ws,
+      })
+    )
     const splitLink = split(
       ({ query }) => {
         const definition = getMainDefinition(query)
@@ -237,30 +209,6 @@ export class QueryNodeApi {
     })
   }
 
-  async dataObjectsByBagId(bagId: string): Promise<DataObjectInfoFragment[]> {
-    return this.multipleEntitiesQuery<GetDataObjectsByBagIdQuery, GetDataObjectsByBagIdQueryVariables>(
-      GetDataObjectsByBagId,
-      { bagId },
-      'storageDataObjects'
-    )
-  }
-
-  async dataObjectsByVideoId(videoId: string): Promise<DataObjectInfoFragment[]> {
-    return this.multipleEntitiesQuery<GetDataObjectsByVideoIdQuery, GetDataObjectsByVideoIdQueryVariables>(
-      GetDataObjectsByVideoId,
-      { videoId },
-      'storageDataObjects'
-    )
-  }
-
-  async dataObjectsByChannelId(channelId: string): Promise<DataObjectInfoFragment[]> {
-    return this.multipleEntitiesQuery<GetDataObjectsByChannelIdQuery, GetDataObjectsByChannelIdQueryVariables>(
-      GetDataObjectsByChannelId,
-      { channelId },
-      'storageDataObjects'
-    )
-  }
-
   async getChannelById(channelId: string): Promise<ChannelFieldsFragment | null> {
     return this.uniqueEntityQuery<GetChannelByIdQuery, GetChannelByIdQueryVariables>(
       GetChannelById,
@@ -296,52 +244,6 @@ export class QueryNodeApi {
     return validNodesInfo
   }
 
-  async storageBucketsForNewChannel(): Promise<StorageNodeInfoFragment[]> {
-    const countQueryResult = await this.uniqueEntityQuery<StorageBucketsCountQuery, StorageBucketsCountQueryVariables>(
-      StorageBucketsCount,
-      {},
-      'storageBucketsConnection'
-    )
-    if (!countQueryResult) {
-      throw Error('Invalid query. Could not fetch storage buckets count information')
-    }
-
-    const buckets = await this.multipleEntitiesQuery<GetStorageBucketsQuery, GetStorageBucketsQueryVariables>(
-      GetStorageBuckets,
-      { count: countQueryResult.totalCount },
-      'storageBuckets'
-    )
-
-    // sorting buckets based on available size, if two buckets have same
-    // available size then sort the two based on available dataObjects count
-    return buckets.sort(
-      (x, y) =>
-        new BN(y.dataObjectsSizeLimit)
-          .sub(new BN(y.dataObjectsSize))
-          .cmp(new BN(x.dataObjectsSizeLimit).sub(new BN(x.dataObjectsSize))) ||
-        new BN(y.dataObjectCountLimit)
-          .sub(new BN(y.dataObjectsCount))
-          .cmp(new BN(x.dataObjectCountLimit).sub(new BN(x.dataObjectsCount)))
-    )
-  }
-
-  async distributionBucketsForNewChannel(): Promise<DistributionBucketFamilyFieldsFragment[]> {
-    return this.multipleEntitiesQuery<
-      GetDistributionFamiliesAndBucketsQuery,
-      GetDistributionFamiliesAndBucketsQueryVariables
-    >(GetDistributionFamiliesAndBuckets, {}, 'distributionBucketFamilies')
-  }
-
-  async membersByIds(ids: MemberId[] | string[]): Promise<MembershipFieldsFragment[]> {
-    return this.multipleEntitiesQuery<GetMembersByIdsQuery, GetMembersByIdsQueryVariables>(
-      GetMembersByIds,
-      {
-        ids: ids.map((id) => id.toString()),
-      },
-      'memberships'
-    )
-  }
-
   async memberById(id: MemberId | string): Promise<MembershipFieldsFragment | null> {
     return this.uniqueEntityQuery<GetMemberByIdQuery, GetMemberByIdQueryVariables>(
       GetMemberById,
@@ -389,29 +291,10 @@ export class QueryNodeApi {
     )
   }
 
-  async getStorageBagInfoForAsset(assetId: string, throwError = true): Promise<string | undefined> {
-    const result = await this.uniqueEntityQuery<
-      GetStorageBagInfoForAssetQuery,
-      GetStorageBagInfoForAssetQueryVariables
-    >(
-      GetStorageBagInfoForAsset,
-      {
-        assetId,
-      },
-      'storageDataObjectByUniqueInput'
-    )
-
-    if (throwError && !result) {
-      throw Error('Could not fetch storage bag information for asset with id: ' + assetId)
-    }
-
-    return result?.storageBagId
-  }
-
   public async getQueryNodeState(): Promise<QueryNodeStateFieldsFragment | null> {
     // fetch cached state
     const cachedState = this.apolloClient.readFragment<
-      QueryNodeStateSubscription['stateSubscription'],
+      QueryNodeStateSubscription['processorState'],
       QueryNodeStateSubscriptionVariables
     >({
       id: 'ProcessorState',
@@ -427,7 +310,7 @@ export class QueryNodeApi {
     return this.uniqueEntitySubscription<QueryNodeStateSubscription, QueryNodeStateSubscriptionVariables>(
       QueryNodeState,
       {},
-      'stateSubscription'
+      'processorState'
     )
   }
 }
diff --git a/src/services/query-node/codegen.yml b/src/services/query-node/codegen.yml
@@ -1,6 +1,6 @@
 overwrite: true
 
-schema: https://query.joystream.org/graphql
+schema: './src/services/query-node/schema.graphql'
 
 documents:
   - './src/services/query-node/queries/*.graphql'