From 342e309ff32873b33ae957f514c5c725400c76bf Mon Sep 17 00:00:00 2001 From: poka <poka@p2p.legal> Date: Tue, 19 Dec 2023 19:51:23 +0100 Subject: [PATCH] feat: optionnal run migrate_csplus on datapod startup --- .dockerignore | 1 + .env.example | 3 +- .gitignore | 2 +- Dockerfile | 13 ++++- docker-compose.prod.yml | 1 + index.ts | 15 ++++- lib/utils.ts | 55 +++++++++++++++++++ load.sh | 2 +- {scrap_csplus => migrate_csplus}/Cargo.lock | 30 +++++----- {scrap_csplus => migrate_csplus}/Cargo.toml | 2 +- .../src/import_only.rs | 0 {scrap_csplus => migrate_csplus}/src/main.rs | 1 + 12 files changed, 104 insertions(+), 21 deletions(-) create mode 100644 .dockerignore rename {scrap_csplus => migrate_csplus}/Cargo.lock (99%) rename {scrap_csplus => migrate_csplus}/Cargo.toml (95%) rename {scrap_csplus => migrate_csplus}/src/import_only.rs (100%) rename {scrap_csplus => migrate_csplus}/src/main.rs (99%) diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..ddb7166 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +migrate_csplus/target/ diff --git a/.env.example b/.env.example index bdba3ab..6382b4b 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,5 @@ DB_USER=datapod DB_DATABASE=datapod_db DB_PASSWORD=my_db_password -HASURA_GRAPHQL_ADMIN_SECRET=my_hasura_password \ No newline at end of file +HASURA_GRAPHQL_ADMIN_SECRET=my_hasura_password +IMPORT_CSPLUS_DATA=false diff --git a/.gitignore b/.gitignore index c59aa15..5662e08 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ .env -scrap_csplus/target/ +migrate_csplus/target/ diff --git a/Dockerfile b/Dockerfile index 60e8456..55eaf0d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,18 @@ +# Step 1: Build Rust +FROM rust:latest as rust-builder + +WORKDIR /usr/src/myapp +COPY migrate_csplus/ ./migrate_csplus + +RUN cargo build --release --manifest-path migrate_csplus/Cargo.toml + +# Step 2: Deno image FROM denoland/deno:alpine WORKDIR /app +COPY --from=rust-builder /usr/src/myapp/migrate_csplus/target/release/migrate_csplus ./migrate_csplus/target/release/migrate_csplus + COPY index.ts . COPY lib ./lib ENV PRODUCTION=true @@ -11,4 +22,4 @@ RUN deno cache index.ts EXPOSE 3000 -CMD ["deno", "run", "--allow-env", "--allow-read", "--allow-write", "--allow-net", "index.ts"] +CMD ["deno", "run", "--allow-env", "--allow-read", "--allow-write", "--allow-net", "--allow-run", "index.ts"] diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index af007f4..65095de 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -13,3 +13,4 @@ services: DB_USER: ${DB_USER} DB_PASSWORD: ${DB_PASSWORD} DB_DATABASE: ${DB_DATABASE} + IMPORT_CSPLUS_DATA: ${IMPORT_CSPLUS_DATA} diff --git a/index.ts b/index.ts index 7ed6ee7..e547b49 100644 --- a/index.ts +++ b/index.ts @@ -2,8 +2,9 @@ import { Application, Context, Router } from "https://deno.land/x/oak@v12.6.1/mo import { Client } from "https://deno.land/x/postgres@v0.17.0/mod.ts"; import { load } from "https://deno.land/std@0.209.0/dotenv/mod.ts"; import { updateProfile } from "./lib/update_profile.ts"; +import { isProfilesTableEmpty, runCsplusImport, waitForTableCreation } from "./lib/utils.ts"; -let dbUser, dbDatabase, dbPassword, dbHostname; +let dbUser, dbDatabase, dbPassword, dbHostname, importCsplusData; const dbPort = 5432; const isProduction = Deno.env.get("PRODUCTION") === "true"; @@ -12,12 +13,14 @@ if (isProduction) { dbUser = Deno.env.get("DB_USER"); dbDatabase = Deno.env.get("DB_DATABASE"); dbPassword = Deno.env.get("DB_PASSWORD"); + importCsplusData = Deno.env.get("IMPORT_CSPLUS_DATA") === "true"; dbHostname = "postgres-datapod"; } else { const env = await load(); dbUser = env["DB_USER"]; dbDatabase = env["DB_DATABASE"]; dbPassword = env["DB_PASSWORD"]; + importCsplusData = env["IMPORT_CSPLUS_DATA"] === "true"; dbHostname = "localhost"; } @@ -33,10 +36,20 @@ await client.connect() const app = new Application(); const router = new Router(); +// Wait for table creation before continue +await waitForTableCreation(client, 'public.profiles'); + +// Import Cs+ data +const profilesEmpty = await isProfilesTableEmpty(client); +if (profilesEmpty && importCsplusData) { + await runCsplusImport(isProduction) +} + // Manage routes router.post("/update-profile-data", async (ctx: Context) => await updateProfile(ctx, client)); app.use(router.routes()); app.use(router.allowedMethods()); +console.log("\nDatapod is started") await app.listen({ port: 3000 }); diff --git a/lib/utils.ts b/lib/utils.ts index 8930e84..eb94d95 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -1,3 +1,5 @@ +import { Client } from "https://deno.land/x/postgres@v0.17.0/client.ts"; + export function convertBase64ToBytea(base64String: string): Uint8Array { // Remove the MIME type prefix from the base64 string, if present const base64Data = base64String.split(',')[1] || base64String; @@ -13,3 +15,56 @@ export function convertBase64ToBytea(base64String: string): Uint8Array { return bytes; } + +export async function runCsplusImport(isProduction: boolean) { + const command = new Deno.Command("./migrate_csplus/target/release/migrate_csplus", { + env: { "PRODUCTION": isProduction.toString() }, + stdout: "piped", + stderr: "piped", + }); + + const process = command.spawn(); + + process.stdout.pipeTo(Deno.stdout.writable, { preventClose: true, preventCancel: true, preventAbort: true}); + process.stderr.pipeTo(Deno.stderr.writable, { preventClose: true, preventCancel: true, preventAbort: true}); + + await process.status; + console.log("End of Cs+ data import") +} + +export async function isProfilesTableEmpty(client: Client): Promise<boolean> { + const result = await client.queryObject<{ count: bigint }>('SELECT COUNT(*) FROM public.profiles'); + return result.rows[0].count === 0n; +} + +interface TableCheckResult { + to_regclass: string | null; +} + +async function checkTableExists(client: Client, tableName: string): Promise<boolean> { + try { + const result = await client.queryObject<TableCheckResult>(`SELECT to_regclass('${tableName}')`); + return result.rows[0].to_regclass !== null; + } catch (error) { + console.error("Error checking table existence:", error); + return false; + } +} +function delay(ms: number): Promise<void> { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +export async function waitForTableCreation(client: Client, tableName: string, maxAttempts = 10) { + let attempts = 0; + while (attempts < maxAttempts) { + if (await checkTableExists(client, tableName)) { + console.log(`Table ${tableName} found.`); + return; + } + + await delay(1000); + attempts++; + } + + throw new Error(`Table ${tableName} not found after ${maxAttempts} try.`); +} diff --git a/load.sh b/load.sh index fd1d8d7..5765edc 100755 --- a/load.sh +++ b/load.sh @@ -6,7 +6,7 @@ if [[ $option == "dev" ]]; then echo "Start datapod in dev mode" docker compose -f docker-compose.yml -f docker-compose.prod.yml -f docker-compose.override.yml down -v docker compose up -d - deno run --allow-env --allow-read --allow-write --allow-net --watch index.ts + deno run --allow-env --allow-read --allow-write --allow-net --allow-run --watch index.ts else echo "Start datapod in production mode" docker compose -f docker-compose.yml -f docker-compose.prod.yml -f docker-compose.override.yml down diff --git a/scrap_csplus/Cargo.lock b/migrate_csplus/Cargo.lock similarity index 99% rename from scrap_csplus/Cargo.lock rename to migrate_csplus/Cargo.lock index f4005a3..49c8e74 100644 --- a/scrap_csplus/Cargo.lock +++ b/migrate_csplus/Cargo.lock @@ -1420,6 +1420,21 @@ dependencies = [ "zeroize", ] +[[package]] +name = "migrate_csplus" +version = "0.1.0" +dependencies = [ + "base64 0.21.5", + "bs58", + "dotenv", + "reqwest", + "serde", + "serde_json", + "sp-core", + "tokio", + "tokio-postgres", +] + [[package]] name = "mime" version = "0.3.17" @@ -2102,21 +2117,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "scrap_csplus" -version = "0.1.0" -dependencies = [ - "base64 0.21.5", - "bs58", - "dotenv", - "reqwest", - "serde", - "serde_json", - "sp-core", - "tokio", - "tokio-postgres", -] - [[package]] name = "secp256k1" version = "0.28.0" diff --git a/scrap_csplus/Cargo.toml b/migrate_csplus/Cargo.toml similarity index 95% rename from scrap_csplus/Cargo.toml rename to migrate_csplus/Cargo.toml index 62830fe..5c612e7 100644 --- a/scrap_csplus/Cargo.toml +++ b/migrate_csplus/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "scrap_csplus" +name = "migrate_csplus" version = "0.1.0" edition = "2018" diff --git a/scrap_csplus/src/import_only.rs b/migrate_csplus/src/import_only.rs similarity index 100% rename from scrap_csplus/src/import_only.rs rename to migrate_csplus/src/import_only.rs diff --git a/scrap_csplus/src/main.rs b/migrate_csplus/src/main.rs similarity index 99% rename from scrap_csplus/src/main.rs rename to migrate_csplus/src/main.rs index 644c8e9..9b4c690 100644 --- a/scrap_csplus/src/main.rs +++ b/migrate_csplus/src/main.rs @@ -250,6 +250,7 @@ async fn fetch_profiles() -> Result<(), Box<dyn Error>> { #[tokio::main] async fn main() { + println!("Starting Cs+ data migration..."); if let Err(e) = fetch_profiles().await { eprintln!("Error: {}", e); } -- GitLab