🧧 Recommendations

.env

@@ -9,6 +9,8 @@ DB_PASS=squid
 DB_ADMIN_USER=admin
 DB_ADMIN_PASS=admin
 DB_PORT=23798
+INTERACTIONS_DB_PORT=23799
+
 # Processor service host
 PROCESSOR_HOST=localhost
 # Processor service prometheus port
@@ -17,6 +19,17 @@ PROCESSOR_PROMETHEUS_PORT=3337
 GQL_PORT=4350
 # Auth api port
 AUTH_API_PORT=4074
+# Recommendation service config
+INTERACTIONS_API_PORT=4075
+# Set private key and database name for recommendation service
+# If you set only private key, Orion will support only local interactions service (for now at least)
+RECOMMENDATION_SERVICE_PRIVATE_KEY=save-interactions-locally
+# jsgenesis-dev
+RECOMMENDATION_SERVICE_DATABASE=
+RECOMMENDATION_SERVICE_DATABASE_REGION=eu-west
+# this will disable export block check for recommendation system - sync all data from the beginning
+FORCE_RECOMMENDATION_DATA_SYNC=false
+
 
 # Archive gateway url
 ARCHIVE_GATEWAY_URL=${CUSTOM_ARCHIVE_GATEWAY_URL:-http://localhost:8888/graphql}

Makefile

@@ -18,13 +18,16 @@ serve:
 serve-auth-api:
 	@npm run auth-server-start
 
+serve-interactions-api:
+	@npm run interactions-server-start
+
 migrate:
 	@npx squid-typeorm-migration apply
 
 dbgen:
 	@npx squid-typeorm-migration generate
 
-generate-migrations: 
+generate-migrations:
 	@docker run -d --name temp_migrations_db \
 		-e POSTGRES_DB=squid \
 		-e POSTGRES_HOST_AUTH_METHOD=trust \
@@ -49,27 +52,37 @@ codegen:
 	@npm run generate:schema || true
 	@npx squid-typeorm-codegen
 
+setup-recommendations:
+	@npm run recommendations:setup
+
 network-tests:
 	@npm run generate:network-tests || true
 
 typegen:
 	@npx squid-substrate-typegen typegen.json
 
-prepare: install typegen codegen build
+prepare: install typegen codegen build setup-recommendations
 
 up-squid:
-	@docker network create joystream_default || true
-	@docker-compose up -d
+	@set -a; \
+	source .env; \
+	set +a; \
+	docker network create joystream_default || true; \
+	if [ -z "$$RECOMMENDATION_SERVICE_PRIVATE_KEY" ] && [ -z "$$RECOMMENDATION_SERVICE_DATABASE"]; then \
+		docker-compose up -d; \
+	else \
+		docker-compose --profile interactions up -d; \
+	fi \
 
 up-archive:
-	@docker network create joystream_default || true
+	@docker network create joystream_default || truee
 	@docker-compose -f archive/docker-compose.yml up -d
 
 up: up-archive up-squid
 
 down-squid:
-	@docker-compose down -v
-	
+	@docker-compose down -v --remove-orphans
+
 down-archive:
 	@docker-compose -f archive/docker-compose.yml down -v
 

docker-compose.yml

@@ -18,6 +18,22 @@ services:
       - orion_db_data:/var/lib/postgresql/data
       - ./db/postgres.conf:/etc/postgresql/postgresql.conf
 
+  interactions_db:
+    image: mongo:7.0.0
+    container_name: interactions_db
+    hostname: interactions_db
+    profiles: ['interactions']
+    restart: unless-stopped
+    shm_size: 1g
+    command: ['mongod', '--port', '${INTERACTIONS_DB_PORT}']
+    ports:
+      - '${INTERACTIONS_DB_PORT}:${INTERACTIONS_DB_PORT}'
+    volumes:
+      - interactions_db_data:/data/db
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: squid
+      MONGO_INITDB_ROOT_PASSWORD: squid
+
   orion_processor:
     container_name: orion_processor
     hostname: orion_processor
@@ -32,6 +48,7 @@ services:
     depends_on:
       - orion_db
     volumes:
+      - processor_queue:/app/persistedQueue
       - type: bind
         source: .
         target: /orion
@@ -52,6 +69,7 @@ services:
     depends_on:
       - orion_db
     volumes:
+      - graphql_queue:/app/persistedQueue
       - type: bind
         source: .
         target: /orion
@@ -74,6 +92,7 @@ services:
     depends_on:
       - orion_db
     volumes:
+      - auth_api_queue:/app/persistedQueue
       - type: bind
         source: .
         target: /orion
@@ -82,8 +101,36 @@ services:
     ports:
       - '${AUTH_API_PORT}:${AUTH_API_PORT}'
 
+  orion_interactions-api:
+    container_name: orion_interactions-api
+    hostname: orion_interactions-api
+    profiles: ['interactions']
+    image: node:18
+    restart: unless-stopped
+    env_file:
+      - .env
+      - docker.env
+    environment:
+      - SQD_TRACE=interactions
+    depends_on:
+      - orion_db
+    volumes:
+      - interactions_api_queue:/app/persistedQueue
+      - type: bind
+        source: .
+        target: /orion
+    working_dir: /orion
+    command: ['make', 'serve-interactions-api']
+    ports:
+      - '${INTERACTIONS_API_PORT}:${INTERACTIONS_API_PORT}'
+
 volumes:
   orion_db_data:
+  interactions_db_data:
+  interactions_api_queue:
+  auth_api_queue:
+  graphql_queue:
+  processor_queue:
 
 networks:
   default:

docs/developer-guide/architecture-overview.md

@@ -32,6 +32,7 @@ The services can be divided into 2 main categories:
     - [Squid processor](https://docs.subsquid.io/basics/squid-processor/): It's responsible for processing Joystream blockchain events and creating, updating or deleting the database entities (such as `Channel` or `Video`) accordingly, so that they remain in-sync with the current Joystream chain state.
     - [GraphQL server](https://docs.subsquid.io/graphql-api/overview/): Orion's main GraphQL server which handles all the queries and mutations.
     - [Auth server](../../src/auth-server/docs/README.md): Orion's authentication server (REST API) responsible for handling user authentication.
+    - [Interactions server](../../src/interactions-server/docs/README.md): Orion's interactions proxy server (REST API) responsible for limiting the amount of interactions sent to the recommendations' system.
 
 
 ## Directory structure and important files

docs/operator-guide/index.md

@@ -22,6 +22,7 @@ To learn how to run your own instance of Atlas, please refer to the [Atlas opera
 - Production deployment:
     - [Deploying Orion to production](./tutorials/deploying-orion.md)
     - [Upgrading Orion](./tutorials/upgrading-orion.md)
+- [Recommendations in Orion](./tutorials/recommendations.md)
 
 
 ## Orion architecture
@@ -46,4 +47,4 @@ To learn how to deploy Orion to production, please read the _[Deployment](./tuto
 
 ### Upgrading a production instance
 
-The process of upgrading a production instance of Orion in the event of a new release is described in the _[Upgrading](./tutorials/upgrading-orion.md)_ tutorial.
+The process of upgrading a production instance of Orion in the event of a new release is described in the _[Upgrading](./tutorials/upgrading-orion.md)_ tutorial.

docs/operator-guide/tutorials/recommendations.md

@@ -0,0 +1,21 @@
+# User recommendations in Orion
+User recommendations are outsourced to the external provider.
+Initial implementation was done with the Recombee API, and it is out of the box solution offered. 
+
+### Abstraction
+All the logic for the provider was abstracted and ecapsulated in [`RecommendationServiceManager`](../../../src/utils/RecommendationServiceManager.ts) this means, that if you want to change 
+the provider you just have to adjust the logic inside it and Orion will use the service to send data and get recommendations.
+
+### Syncing data
+- Video data gets upserted only through processor
+- User data get upserted through `auth-server`
+- Interactions data is proxied thorugh `interactions-server` where the request are limited both by IP and interactions.
+Only iteraction that is not coming thorugh the server is rating interaction (like, dislike), which will be synced through processor on MemberRemark event.
+
+### Env variables
+Recombee client is constructed with 2 variables `RECOMMENDATION_SERVICE_PRIVATE_KEY` and `RECOMMENDATION_SERVICE_DATABASE` both self-explanatory.
+`FORCE_RECOMMENDATION_DATA_SYNC` variable is a boolean that indicate whether processor should consider export block height in terms of syncing videos and ratings with recommendations service. 
+If set to true - it will sync all the data for the first block even if last exported block is greater.
+
+If those two values are provided, when you use `make prepare` command, it will set up item properties and segments supported by Orion.
+If you want to run set up script directly use `make setup-recommendations`.