feat: optionnal run migrate_csplus on datapod startup

.dockerignore

+migrate_csplus/target/

.env.example

@@ -2,3 +2,4 @@ DB_USER=datapod
 DB_DATABASE=datapod_db
 DB_PASSWORD=my_db_password
 HASURA_GRAPHQL_ADMIN_SECRET=my_hasura_password
+IMPORT_CSPLUS_DATA=false

.gitignore

 .env
-scrap_csplus/target/
+migrate_csplus/target/

Dockerfile

+# Step 1: Build Rust
+FROM rust:latest as rust-builder
+
+WORKDIR /usr/src/myapp
+COPY migrate_csplus/ ./migrate_csplus
+
+RUN cargo build --release --manifest-path migrate_csplus/Cargo.toml
+
+# Step 2: Deno image
 FROM denoland/deno:alpine
 
 WORKDIR /app
 
+COPY --from=rust-builder /usr/src/myapp/migrate_csplus/target/release/migrate_csplus ./migrate_csplus/target/release/migrate_csplus
+
 COPY index.ts .
 COPY lib ./lib
 ENV PRODUCTION=true
@@ -11,4 +22,4 @@ RUN deno cache index.ts
 
 EXPOSE 3000
 
-CMD ["deno", "run", "--allow-env", "--allow-read", "--allow-write", "--allow-net", "index.ts"]
+CMD ["deno", "run", "--allow-env", "--allow-read", "--allow-write", "--allow-net", "--allow-run", "index.ts"]

docker-compose.prod.yml

@@ -13,3 +13,4 @@ services:
       DB_USER: ${DB_USER}
       DB_PASSWORD: ${DB_PASSWORD}
       DB_DATABASE: ${DB_DATABASE}
+      IMPORT_CSPLUS_DATA: ${IMPORT_CSPLUS_DATA}

index.ts

@@ -2,8 +2,9 @@ import { Application, Context, Router } from "https://deno.land/x/oak@v12.6.1/mo
 import { Client } from "https://deno.land/x/postgres@v0.17.0/mod.ts";
 import { load } from "https://deno.land/std@0.209.0/dotenv/mod.ts";
 import { updateProfile } from "./lib/update_profile.ts";
+import { isProfilesTableEmpty, runCsplusImport, waitForTableCreation } from "./lib/utils.ts";
 
-let dbUser, dbDatabase, dbPassword, dbHostname;
+let dbUser, dbDatabase, dbPassword, dbHostname, importCsplusData;
 const dbPort = 5432;
 
 const isProduction = Deno.env.get("PRODUCTION") === "true";
@@ -12,12 +13,14 @@ if (isProduction) {
     dbUser = Deno.env.get("DB_USER");
     dbDatabase = Deno.env.get("DB_DATABASE");
     dbPassword = Deno.env.get("DB_PASSWORD");
+    importCsplusData = Deno.env.get("IMPORT_CSPLUS_DATA") === "true";
     dbHostname = "postgres-datapod";
 } else {
     const env = await load();
     dbUser = env["DB_USER"];
     dbDatabase = env["DB_DATABASE"];
     dbPassword = env["DB_PASSWORD"];
+    importCsplusData = env["IMPORT_CSPLUS_DATA"] === "true";
     dbHostname = "localhost";
 }
 
@@ -33,10 +36,20 @@ await client.connect()
 const app = new Application();
 const router = new Router();
 
+// Wait for table creation before continue
+await waitForTableCreation(client, 'public.profiles');
+
+// Import Cs+ data
+const profilesEmpty = await isProfilesTableEmpty(client);
+if (profilesEmpty && importCsplusData) {
+    await runCsplusImport(isProduction)
+}
+
 // Manage routes
 router.post("/update-profile-data", async (ctx: Context) => await updateProfile(ctx, client));
 
 app.use(router.routes());
 app.use(router.allowedMethods());
 
+console.log("\nDatapod is started")
 await app.listen({ port: 3000 });

lib/utils.ts

+import { Client } from "https://deno.land/x/postgres@v0.17.0/client.ts";
+
 export function convertBase64ToBytea(base64String: string): Uint8Array {
     // Remove the MIME type prefix from the base64 string, if present
     const base64Data = base64String.split(',')[1] || base64String;
@@ -13,3 +15,56 @@ export function convertBase64ToBytea(base64String: string): Uint8Array {
 
     return bytes;
 }
+
+export async function runCsplusImport(isProduction: boolean) {
+    const command = new Deno.Command("./migrate_csplus/target/release/migrate_csplus", {
+        env: { "PRODUCTION": isProduction.toString() },
+        stdout: "piped",
+        stderr: "piped",
+    });
+
+    const process = command.spawn();
+    
+    process.stdout.pipeTo(Deno.stdout.writable, { preventClose: true, preventCancel: true, preventAbort: true});
+    process.stderr.pipeTo(Deno.stderr.writable, { preventClose: true, preventCancel: true, preventAbort: true});
+
+    await process.status;
+    console.log("End of Cs+ data import")
+}
+
+export async function isProfilesTableEmpty(client: Client): Promise<boolean> {
+    const result = await client.queryObject<{ count: bigint }>('SELECT COUNT(*) FROM public.profiles');
+    return result.rows[0].count === 0n;
+}
+
+interface TableCheckResult {
+    to_regclass: string | null;
+}
+
+async function checkTableExists(client: Client, tableName: string): Promise<boolean> {
+    try {
+        const result = await client.queryObject<TableCheckResult>(`SELECT to_regclass('${tableName}')`);
+        return result.rows[0].to_regclass !== null;
+    } catch (error) {
+        console.error("Error checking table existence:", error);
+        return false;
+    }
+}
+function delay(ms: number): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+export async function waitForTableCreation(client: Client, tableName: string, maxAttempts = 10) {
+    let attempts = 0;
+    while (attempts < maxAttempts) {
+        if (await checkTableExists(client, tableName)) {
+            console.log(`Table ${tableName} found.`);
+            return;
+        }
+
+        await delay(1000);
+        attempts++;
+    }
+
+    throw new Error(`Table ${tableName} not found after ${maxAttempts} try.`);
+}

load.sh

@@ -6,7 +6,7 @@ if [[ $option == "dev" ]]; then
     echo "Start datapod in dev mode"
     docker compose -f docker-compose.yml -f docker-compose.prod.yml -f docker-compose.override.yml down -v
     docker compose up -d
-    deno run --allow-env --allow-read --allow-write --allow-net --watch index.ts
+    deno run --allow-env --allow-read --allow-write --allow-net --allow-run --watch index.ts
 else
     echo "Start datapod in production mode"
     docker compose -f docker-compose.yml -f docker-compose.prod.yml -f docker-compose.override.yml down

scrap_csplus/Cargo.lock → migrate_csplus/Cargo.lock

@@ -1420,6 +1420,21 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "migrate_csplus"
+version = "0.1.0"
+dependencies = [
+ "base64 0.21.5",
+ "bs58",
+ "dotenv",
+ "reqwest",
+ "serde",
+ "serde_json",
+ "sp-core",
+ "tokio",
+ "tokio-postgres",
+]
+
 [[package]]
 name = "mime"
 version = "0.3.17"
@@ -2102,21 +2117,6 @@ version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
 
-[[package]]
-name = "scrap_csplus"
-version = "0.1.0"
-dependencies = [
- "base64 0.21.5",
- "bs58",
- "dotenv",
- "reqwest",
- "serde",
- "serde_json",
- "sp-core",
- "tokio",
- "tokio-postgres",
-]
-
 [[package]]
 name = "secp256k1"
 version = "0.28.0"

scrap_csplus/Cargo.toml → migrate_csplus/Cargo.toml

 [package]
-name = "scrap_csplus"
+name = "migrate_csplus"
 version = "0.1.0"
 edition = "2018"
 

scrap_csplus/src/main.rs → migrate_csplus/src/main.rs

@@ -250,6 +250,7 @@ async fn fetch_profiles() -> Result<(), Box<dyn Error>> {
 
 #[tokio::main]
 async fn main() {
+    println!("Starting Cs+ data migration...");
     if let Err(e) = fetch_profiles().await {
         eprintln!("Error: {}", e);
     }