Merge pull request #322 from zeeshanakram3/connect-to-proxy-instance-…

…using-private-ip

Release: `v3.4.1`

CHANGELOG.md

@@ -1,3 +1,9 @@
+### 3.5.0
+
+- Add support for setting `isShort` field in the video metadata (indicating whether video is a short format, vertical video or not) when creating the video.
+- Support connecting to chisel server proxy ec2 instance using Private IP address.
+- **FIX**: Properly handle error when failing to download the members-only content from Youtube.
+
 ### 3.4.0
 
 - Enable opentelemetry integration for tracing

package.json

@@ -1,6 +1,6 @@
 {
   "name": "youtube-sync",
-  "version": "3.4.0",
+  "version": "3.5.0",
   "license": "MIT",
   "scripts": {
     "postpack": "rm -f oclif.manifest.json",
@@ -84,7 +84,7 @@
     "@google-cloud/monitoring": "^3.0.4",
     "@googleapis/youtube": "^2.0.0",
     "@joystream/cli": "^0.10.0",
-    "@joystream/js": "^1.4.0",
+    "@joystream/js": "^1.8.0",
     "@joystream/prettier-config": "^1.0.0",
     "@joystream/types": "0.20.5",
     "@nestjs/common": "^8.0.0",

socks5-proxy/SETUP.md

@@ -6,7 +6,7 @@ This setup is designed to automate the restarting of an Amazon EC2 instance and
 
 ## Components
 
-1. **Amazon EC2 Instance**: A server that will be stopped and started by the script running on the client machine, resulting in a new public IP address assignment of EC2 proxy server instance.
+1. **Amazon EC2 Instance**: A server that will be stopped and started by the script running on the client machine (YT-synch instance), resulting in a new public IP address assignment of EC2 proxy server instance. _IMPORTANT:_ The EC2 proxy server instance should be created within the same VPC and Subnet as the YT-synch server instance, since YT-synch server instance connects with the proxy server instance using Private IP address.
 
 2. **Chisel**: A fast TCP tunnel over HTTP. In this setup, a Chisel client is run in a Docker container, which connects to a Chisel server running on the EC2 instance.
 

socks5-proxy/start-proxy-client.sh

@@ -13,7 +13,7 @@ set +a
 
 # Fetch the current public IP of the EC2 instance
 echo -e "Fetching new assigned IP address of proxy server instance... \n"
-IP_ADDRESS=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --query "Reservations[*].Instances[*].PublicIpAddress" --output text)
+IP_ADDRESS=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --query "Reservations[*].Instances[*].PrivateIpAddress" --output text)
 
 # Check if IP address is fetched
 if [ -z "$IP_ADDRESS" ]; then

src/repository/video.ts

@@ -95,6 +95,9 @@ function videoRepository(tablePrefix: ResourcePrefix) {
         },
       },
 
+      // Whether video is a short format, vertical video (e.g. Youtube Shorts, TikTok, Instagram Reels)
+      isShort: Boolean,
+
       // Video creation date on youtube
       publishedAt: String,
     },

src/services/httpApi/controllers/membership.ts

@@ -34,7 +34,7 @@ export class MembershipController {
       // Only allow maximum of 5 captcha-free memberships to be created by YT-synch
       // as avoiding this check leads sybil attack where any YPP verified user/channel
       // can created infinitely many memberships causing faucet funds to exhaust.
-      if (user.joystreamMemberIds.length >= 5) {
+      if (user.joystreamMemberIds.length >= 2) {
         throw new Error(`Already created Joysteam memberships ${user.joystreamMemberIds} for user ${user.id}`)
       }
 

src/services/runtime/client.ts

@@ -199,6 +199,7 @@ export class JoystreamClient {
       category: video.category,
       language: video.languageIso,
       isPublic: true,
+      isShort: video.isShort,
       duration: video.duration,
       license: getVideoLicense(video),
       publishedBeforeJoystream: { isPublished: true, date: video.publishedAt },

src/services/syncProcessing/ContentDownloadService.ts

@@ -123,6 +123,7 @@ export class ContentDownloadService {
         { message: 'The downloaded file is empty', code: VideoUnavailableReasons.EmptyDownload },
         { message: 'This video is private', code: VideoUnavailableReasons.Private },
         { message: 'removed by the uploader', code: VideoUnavailableReasons.Private },
+        { message: 'Join this channel to get access to members-only content', code: VideoUnavailableReasons.Private },
         { message: 'size cap for historical videos', code: VideoUnavailableReasons.Skipped },
       ]
 

src/services/syncProcessing/YoutubePollingService.ts

@@ -175,10 +175,7 @@ export class YoutubePollingService {
       }
 
       //  get all videos that are not yet being tracked
-      const untrackedVideos = await this.youtubeApi.ytdlpClient.getVideos(
-        channel,
-        untrackedVideosIds.map((v) => v.id)
-      )
+      const untrackedVideos = await this.youtubeApi.ytdlpClient.getVideos(channel, untrackedVideosIds)
 
       // save all new videos to DB including
       await this.dynamodbService.repo.videos.upsertAll(untrackedVideos)

src/services/youtube/api.ts

@@ -23,7 +23,7 @@ import Schema$Channel = youtube_v3.Schema$Channel
 
 export interface IOpenYTApi {
   ensureChannelExists(channelId: string): Promise<string>
-  getVideos(channel: YtChannel, ids: string[]): Promise<YtVideo[]>
+  getVideos(channel: YtChannel, ids: YtDlpFlatPlaylistOutput): Promise<YtVideo[]>
 }
 
 export class YtDlpClient implements IOpenYTApi {
@@ -41,19 +41,23 @@ export class YtDlpClient implements IOpenYTApi {
       .stdout
   }
 
-  async getVideos(channel: YtChannel, ids: string[]): Promise<YtVideo[]> {
+  async getVideos(channel: YtChannel, ids: YtDlpFlatPlaylistOutput): Promise<YtVideo[]> {
     const videosMetadata: YtDlpVideoOutput[] = []
     const idsChunks = _.chunk(ids, 50)
 
     for (const idsChunk of idsChunks) {
       const videosMetadataChunk = (
         await Promise.all(
-          idsChunk.map(async (id) => {
+          idsChunk.map(async ({ id, isShort }) => {
             try {
               const { stdout } = await this.exec(`${this.ytdlpPath} -J https://www.youtube.com/watch?v=${id}`)
-              return JSON.parse(stdout) as YtDlpVideoOutput
+              return { ...JSON.parse(stdout), isShort } as YtDlpVideoOutput
             } catch (err) {
-              if (err instanceof Error && err.message.includes(`This video is age-restricted`)) {
+              if (
+                err instanceof Error &&
+                (err.message.includes(`This video is age-restricted`) ||
+                  err.message.includes(`Join this channel to get access to members-only content`))
+              ) {
                 return
               }
               throw err
@@ -94,6 +98,7 @@ export class YtDlpClient implements IOpenYTApi {
               video?.license === 'Creative Commons Attribution license (reuse allowed)' ? 'creativeCommon' : 'youtube',
             duration: video?.duration,
             container: video?.ext,
+            isShort: video.isShort,
             viewCount: video?.view_count || 0,
             state: 'New',
           }
@@ -128,12 +133,17 @@ export class YtDlpClient implements IOpenYTApi {
           JSON.parse(stdout).entries.forEach((category: any) => {
             if (category.entries) {
               category.entries.forEach((video: any) => {
-                videos.push({ id: video.id, publishedAt: new Date(video.timestamp * 1000) /** Convert UNIX to date */ })
+                videos.push({
+                  id: video.id,
+                  publishedAt: new Date(video.timestamp * 1000) /** Convert UNIX to date */,
+                  isShort: type === 'shorts',
+                })
               })
             } else {
               videos.push({
                 id: category.id,
                 publishedAt: new Date(category.timestamp * 1000) /** Convert UNIX to date */,
+                isShort: type === 'shorts',
               })
             }
           })

src/types/youtube.ts

@@ -311,6 +311,9 @@ export class YtVideo {
   // joystream video ID in `VideoCreated` event response, returned from joystream runtime after creating a video
   joystreamVideo: JoystreamVideo
 
+  // Whether video is a short format, vertical video (e.g. Youtube Shorts, TikTok, Instagram Reels)
+  isShort: boolean
+
   // Youtube video creation date
   publishedAt: string
 
@@ -368,6 +371,7 @@ export type UploadJobData = YtVideo & {
 export type YtDlpFlatPlaylistOutput = {
   id: string
   publishedAt: Date
+  isShort: boolean
 }[]
 
 export type YtDlpVideoOutput = {
@@ -390,6 +394,10 @@ export type YtDlpVideoOutput = {
   thumbnails: {
     url: string
   }[]
+
+  // This property isn't really part of the YtDlpVideoOutput, but is separately
+  // set based on wether video was downloaded from channels 'shorts' tab or not
+  isShort: boolean
 }
 
 export type FaucetRegisterMembershipParams = {

yarn.lock

@@ -2317,12 +2317,12 @@
     eslint-plugin-react-hooks "^4.0.8"
     eslint-plugin-standard "^4.0.1"
 
-"@joystream/js@^1.4.0":
-  version "1.4.0"
-  resolved "https://registry.yarnpkg.com/@joystream/js/-/js-1.4.0.tgz#deb16175efadfcb131821ab781317f1f7463f620"
-  integrity sha512-kEiKPIhsuk4B2vteAZoFYoZ+slyAiDXu1hDKMBddt1AKWfD1qxrHKAuF2cBkQIN4KpeNEJrRK9vy8G3glvaDvQ==
+"@joystream/js@^1.8.0":
+  version "1.8.0"
+  resolved "https://registry.yarnpkg.com/@joystream/js/-/js-1.8.0.tgz#5fd20754e87371369083b4904173992374e24657"
+  integrity sha512-gIFpNJss0G8PjLsxsOsCE/m4kOlmBU70Q/oSBkC8KpAkjpeqkAAIElHSt33V/F4z3+1pHVBHA8QihzqOCzkQrA==
   dependencies:
-    "@joystream/metadata-protobuf" "^2.8.1"
+    "@joystream/metadata-protobuf" "^2.13.0"
     "@joystream/types" "^2.0.0"
     "@polkadot/util-crypto" "9.5.1"
     axios "^1.2.1"
@@ -2332,7 +2332,21 @@
     merkletreejs "^0.3.9"
     protobufjs "^6.11.3"
 
-"@joystream/metadata-protobuf@^2.5.0", "@joystream/metadata-protobuf@^2.8.1":
+"@joystream/metadata-protobuf@^2.13.0":
+  version "2.13.0"
+  resolved "https://registry.yarnpkg.com/@joystream/metadata-protobuf/-/metadata-protobuf-2.13.0.tgz#74562336fdc1f2860c8b4106759d6511dc6dc5d9"
+  integrity sha512-L1VGCVsR/CWzl7IfYFKZwlKBAL/JPlgtpdyHmcceLgUsJiVsw0v9zkFQvBwe40J3LHy56arDiaiua0G+nn4JXg==
+  dependencies:
+    "@types/iso-3166-2" "^1.0.0"
+    "@types/long" "^4.0.1"
+    google-protobuf "^3.14.0"
+    i18n-iso-countries "^6.8.0"
+    iso-3166-2 "^1.0.0"
+    iso-639-1 "^2.1.9"
+    long "^4.0.0"
+    protobufjs "^6.11.2"
+
+"@joystream/metadata-protobuf@^2.5.0":
   version "2.8.1"
   resolved "https://registry.yarnpkg.com/@joystream/metadata-protobuf/-/metadata-protobuf-2.8.1.tgz#660f5944409dad8aa6695baa0aa86b94b18729de"
   integrity sha512-ckxvQP3RC8gKCJWU1xpXosxEfgFcChgaIncy06AZfn50x6+9mFEsxQTTogN7b1g1T026oSFYZMDq52tkBw2Zew==