Merge pull request #301 from zeeshanakram3/ypp_sync_improvements

YPP sync improvements + other minor fixes
Joystream · Nov 9, 2023 · 5eb03e6 · 5eb03e6
2 parents 9dadb6c + 21cb21c
commit 5eb03e6
Show file tree

Hide file tree

Showing 20 changed files with 285 additions and 121 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,11 @@
+### 3.2.0
+
+- Optimizes Youtube API quota usage by using `yt-dlp` instead of youtubeApi to fetch the video details
+- add `pendingDownloadTimeoutSec` config option to manage timeout for youtube downloads
+- Add multiple variants to `YtVideo.state.VideoUnavailable`
+- **FIX**: add optional locking feature for Dynamodb tables
+- **FIX**: Only calculate priority for sync-able videos
+
 ### 3.1.0
 
 - bump `@bull-board` npm dependency

diff --git a/config.yml b/config.yml
@@ -17,6 +17,7 @@ sync:
     maxConcurrentDownloads: 50
     maxConcurrentUploads: 50
     createVideoTxBatchSize: 10
+    pendingDownloadTimeoutSec: 600 # 10 mins
     storage: 100G
 logs:
   file:

diff --git a/...tion-related-settings-properties-limits-properties-pendingdownloadtimeoutsec.md b/...tion-related-settings-properties-limits-properties-pendingdownloadtimeoutsec.md
@@ -0,0 +1,7 @@
+## pendingDownloadTimeoutSec Type
+
+`integer`
+
+## pendingDownloadTimeoutSec Constraints
+
+**minimum**: the value of this number must greater than or equal to: `60`
diff --git a/...nition-properties-yt-synch-syncronization-related-settings-properties-limits.md b/...nition-properties-yt-synch-syncronization-related-settings-properties-limits.md
@@ -4,27 +4,28 @@
 
 # limits Properties
 
-| Property                                          | Type     | Required | Nullable       | Defined by                                                                                                                                                                                                                                                                                                                        |
-| :------------------------------------------------ | :------- | :------- | :------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| [dailyApiQuota](#dailyapiquota)                   | `object` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-specifies-daily-youtube-api-quota-rationing-scheme-for-youtube-partner-program.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/dailyApiQuota") |
-| [maxConcurrentDownloads](#maxconcurrentdownloads) | `number` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-maxconcurrentdownloads.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/maxConcurrentDownloads")                                                |
-| [createVideoTxBatchSize](#createvideotxbatchsize) | `number` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-createvideotxbatchsize.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/createVideoTxBatchSize")                                                |
-| [maxConcurrentUploads](#maxconcurrentuploads)     | `number` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-maxconcurrentuploads.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/maxConcurrentUploads")                                                    |
-| [storage](#storage)                               | `string` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-storage.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/storage")                                                                              |
+| Property                                                | Type      | Required | Nullable       | Defined by                                                                                                                                                                                                                                                                                                                        |
+| :------------------------------------------------------ | :-------- | :------- | :------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [dailyApiQuota](#dailyapiquota)                         | `object`  | Required | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-specifies-daily-youtube-api-quota-rationing-scheme-for-youtube-partner-program.md 'https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/dailyApiQuota') |
+| [maxConcurrentDownloads](#maxconcurrentdownloads)       | `number`  | Required | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-maxconcurrentdownloads.md 'https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/maxConcurrentDownloads')                                                |
+| [createVideoTxBatchSize](#createvideotxbatchsize)       | `number`  | Required | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-createvideotxbatchsize.md 'https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/createVideoTxBatchSize')                                                |
+| [maxConcurrentUploads](#maxconcurrentuploads)           | `number`  | Required | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-maxconcurrentuploads.md 'https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/maxConcurrentUploads')                                                    |
+| [pendingDownloadTimeoutSec](#pendingdownloadtimeoutsec) | `integer` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-pendingdownloadtimeoutsec.md 'https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/pendingDownloadTimeoutSec')                                          |
+| [storage](#storage)                                     | `string`  | Required | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-storage.md 'https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/storage')                                                                              |
 
 ## dailyApiQuota
 
 Specifies daily Youtube API quota rationing scheme for Youtube Partner Program
 
 `dailyApiQuota`
 
-*   is required
+- is required
 
-*   Type: `object` ([Specifies daily Youtube API quota rationing scheme for Youtube Partner Program](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-specifies-daily-youtube-api-quota-rationing-scheme-for-youtube-partner-program.md))
+- Type: `object` ([Specifies daily Youtube API quota rationing scheme for Youtube Partner Program](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-specifies-daily-youtube-api-quota-rationing-scheme-for-youtube-partner-program.md))
 
-*   cannot be null
+- cannot be null
 
-*   defined in: [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-specifies-daily-youtube-api-quota-rationing-scheme-for-youtube-partner-program.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/dailyApiQuota")
+- defined in: [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-specifies-daily-youtube-api-quota-rationing-scheme-for-youtube-partner-program.md 'https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/dailyApiQuota')
 
 ### dailyApiQuota Type
 
@@ -36,13 +37,13 @@ Max no. of videos that should be concurrently downloaded from Youtube to be prep
 
 `maxConcurrentDownloads`
 
-*   is required
+- is required
 
-*   Type: `number`
+- Type: `number`
 
-*   cannot be null
+- cannot be null
 
-*   defined in: [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-maxconcurrentdownloads.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/maxConcurrentDownloads")
+- defined in: [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-maxconcurrentdownloads.md 'https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/maxConcurrentDownloads')
 
 ### maxConcurrentDownloads Type
 
@@ -62,13 +63,13 @@ No. of videos that should be created in a batched 'create_video' tx
 
 `createVideoTxBatchSize`
 
-*   is required
+- is required
 
-*   Type: `number`
+- Type: `number`
 
-*   cannot be null
+- cannot be null
 
-*   defined in: [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-createvideotxbatchsize.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/createVideoTxBatchSize")
+- defined in: [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-createvideotxbatchsize.md 'https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/createVideoTxBatchSize')
 
 ### createVideoTxBatchSize Type
 
@@ -88,13 +89,13 @@ Max no. of videos that should be concurrently uploaded to Joystream's storage no
 
 `maxConcurrentUploads`
 
-*   is required
+- is required
 
-*   Type: `number`
+- Type: `number`
 
-*   cannot be null
+- cannot be null
 
-*   defined in: [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-maxconcurrentuploads.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/maxConcurrentUploads")
+- defined in: [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-maxconcurrentuploads.md 'https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/maxConcurrentUploads')
 
 ### maxConcurrentUploads Type
 
@@ -108,30 +109,52 @@ The default value is:
 50
 ```
 
+## pendingDownloadTimeoutSec
+
+Timeout for pending youtube video downloads in seconds
+
+`pendingDownloadTimeoutSec`
+
+- is required
+
+- Type: `integer`
+
+- cannot be null
+
+- defined in: [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-pendingdownloadtimeoutsec.md 'https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/pendingDownloadTimeoutSec')
+
+### pendingDownloadTimeoutSec Type
+
+`integer`
+
+### pendingDownloadTimeoutSec Constraints
+
+**minimum**: the value of this number must greater than or equal to: `60`
+
 ## storage
 
 Maximum total size of all downloaded assets stored in `downloadsDir`
 
 `storage`
 
-*   is required
+- is required
 
-*   Type: `string`
+- Type: `string`
 
-*   cannot be null
+- cannot be null
 
-*   defined in: [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-storage.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/storage")
+- defined in: [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-storage.md 'https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/storage')
 
 ### storage Type
 
 `string`
 
 ### storage Constraints
 
-**pattern**: the string must match the following regular expression: 
+**pattern**: the string must match the following regular expression:
 
 ```regexp
 ^[0-9]+(B|K|M|G|T)$
 ```
 
-[try pattern](https://regexr.com/?expression=%5E%5B0-9%5D%2B\(B%7CK%7CM%7CG%7CT\)%24 "try regular expression with regexr.com")
+[try pattern](<https://regexr.com/?expression=%5E%5B0-9%5D%2B(B%7CK%7CM%7CG%7CT)%24> 'try regular expression with regexr.com')
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "youtube-sync",
-  "version": "3.1.0",
+  "version": "3.2.0",
   "license": "MIT",
   "scripts": {
     "postpack": "rm -f oclif.manifest.json",

diff --git a/src/app/index.ts b/src/app/index.ts
@@ -48,6 +48,7 @@ export class Service {
         this.logging,
         this.dynamodbService,
         this.youtubeApi,
+        this.runtimeApi,
         this.joystreamClient,
         this.queryNodeApi
       )

diff --git a/src/repository/stats.ts b/src/repository/stats.ts
@@ -29,9 +29,19 @@ export class StatsRepository implements IRepository<Stats> {
   // lock any updates on video table
   private readonly ASYNC_LOCK_ID = 'stat'
   private asyncLock: AsyncLock = new AsyncLock({ maxPending: Number.MAX_SAFE_INTEGER })
+  private useLock: boolean // Flag to determine if locking should be used
 
-  constructor(tablePrefix: ResourcePrefix) {
+  constructor(tablePrefix: ResourcePrefix, useLock: boolean = true) {
     this.model = statsRepository(tablePrefix)
+    this.useLock = useLock
+  }
+
+  private async withLock<T>(func: () => Promise<T>): Promise<T> {
+    if (this.useLock) {
+      return this.asyncLock.acquire(this.ASYNC_LOCK_ID, func)
+    } else {
+      return func()
+    }
   }
 
   async getModel() {
@@ -66,7 +76,7 @@ export class StatsRepository implements IRepository<Stats> {
   }
 
   async scan(init: ConditionInitializer, f: (q: Scan<AnyItem>) => Scan<AnyItem>): Promise<Stats[]> {
-    return this.asyncLock.acquire(this.ASYNC_LOCK_ID, async () => {
+    return this.withLock(async () => {
       let lastKey = undefined
       const results = []
       do {
@@ -82,14 +92,14 @@ export class StatsRepository implements IRepository<Stats> {
   }
 
   async get(date: string): Promise<Stats | undefined> {
-    return this.asyncLock.acquire(this.ASYNC_LOCK_ID, async () => {
+    return this.withLock(async () => {
       const result = await this.model.get({ partition: 'stats', date })
       return result ? mapTo<Stats>(result) : undefined
     })
   }
 
   async save(model: Stats): Promise<Stats> {
-    return this.asyncLock.acquire(this.ASYNC_LOCK_ID, async () => {
+    return this.withLock(async () => {
       const update = omit(['id', 'updatedAt'], model)
       const result = await this.model.update({ partition: 'stats', date: model.date }, update)
       return mapTo<Stats>(result)
@@ -101,7 +111,7 @@ export class StatsRepository implements IRepository<Stats> {
   }
 
   async query(init: ConditionInitializer, f: (q: Query<AnyItem>) => Query<AnyItem>) {
-    return this.asyncLock.acquire(this.ASYNC_LOCK_ID, async () => {
+    return this.withLock(async () => {
       let lastKey = undefined
       const results = []
       do {

diff --git a/src/repository/user.ts b/src/repository/user.ts
@@ -74,9 +74,19 @@ export class UsersRepository implements IRepository<YtUser> {
   // lock any updates on video table
   private readonly ASYNC_LOCK_ID = 'user'
   private asyncLock: AsyncLock = new AsyncLock({ maxPending: Number.MAX_SAFE_INTEGER })
+  private useLock: boolean // Flag to determine if locking should be used
 
-  constructor(tablePrefix: ResourcePrefix) {
+  constructor(tablePrefix: ResourcePrefix, useLock: boolean = true) {
     this.model = createUserModel(tablePrefix)
+    this.useLock = useLock
+  }
+
+  private async withLock<T>(func: () => Promise<T>): Promise<T> {
+    if (this.useLock) {
+      return this.asyncLock.acquire(this.ASYNC_LOCK_ID, func)
+    } else {
+      return func()
+    }
   }
 
   async upsertAll(users: YtUser[]): Promise<YtUser[]> {
@@ -85,7 +95,7 @@ export class UsersRepository implements IRepository<YtUser> {
   }
 
   async scan(init: ConditionInitializer, f: (q: Scan<AnyItem>) => Scan<AnyItem>): Promise<YtUser[]> {
-    return this.asyncLock.acquire(this.ASYNC_LOCK_ID, async () => {
+    return this.withLock(async () => {
       let lastKey = undefined
       const results = []
       do {
@@ -101,29 +111,29 @@ export class UsersRepository implements IRepository<YtUser> {
   }
 
   async get(id: string): Promise<YtUser | undefined> {
-    return this.asyncLock.acquire(this.ASYNC_LOCK_ID, async () => {
+    return this.withLock(async () => {
       const result = await this.model.get({ id })
       return result ? mapTo<YtUser>(result) : undefined
     })
   }
 
   async save(user: YtUser): Promise<YtUser> {
-    return this.asyncLock.acquire(this.ASYNC_LOCK_ID, async () => {
+    return this.withLock(async () => {
       const update = omit(['id', 'updatedAt'], user)
       const result = await this.model.update({ id: user.id }, update)
       return mapTo<YtUser>(result)
     })
   }
 
   async delete(id: string): Promise<void> {
-    return this.asyncLock.acquire(this.ASYNC_LOCK_ID, async () => {
+    return this.withLock(async () => {
       await this.model.delete({ id })
       return
     })
   }
 
   async query(init: ConditionInitializer, f: (q: Query<AnyItem>) => Query<AnyItem>) {
-    return this.asyncLock.acquire(this.ASYNC_LOCK_ID, async () => {
+    return this.withLock(async () => {
       let lastKey = undefined
       const results = []
       do {

diff --git a/src/repository/video.ts b/src/repository/video.ts
@@ -95,9 +95,6 @@ function videoRepository(tablePrefix: ResourcePrefix) {
         },
       },
 
-      // ID of the corresponding Joystream Channel (De-normalized from Channel table)
-      joystreamChannelId: Number,
-
       // Video creation date on youtube
       publishedAt: String,
     },