diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000000000000000000000000000000000..ddb71668aec41e2a74b4ea49d803cbd12c3feb2f --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +migrate_csplus/target/ diff --git a/.env.example b/.env.example index bdba3aba349ae4310eb2072903abe0ed9e3e5b09..6382b4be698ca3faacab61f2d219dbc3f90cb455 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,5 @@ DB_USER=datapod DB_DATABASE=datapod_db DB_PASSWORD=my_db_password -HASURA_GRAPHQL_ADMIN_SECRET=my_hasura_password \ No newline at end of file +HASURA_GRAPHQL_ADMIN_SECRET=my_hasura_password +IMPORT_CSPLUS_DATA=false diff --git a/.gitignore b/.gitignore index c59aa153e1e51ace80a4fa65b5470421e7c9f6aa..5662e083f478e8b61dc31a8032711a5238b922a9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ .env -scrap_csplus/target/ +migrate_csplus/target/ diff --git a/Dockerfile b/Dockerfile index 60e84562eb5a7cdfb941fdd7a7dc5bf63178282d..55eaf0df1a09d12b76c5f8cb82804a172062dc7b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,18 @@ +# Step 1: Build Rust +FROM rust:latest as rust-builder + +WORKDIR /usr/src/myapp +COPY migrate_csplus/ ./migrate_csplus + +RUN cargo build --release --manifest-path migrate_csplus/Cargo.toml + +# Step 2: Deno image FROM denoland/deno:alpine WORKDIR /app +COPY --from=rust-builder /usr/src/myapp/migrate_csplus/target/release/migrate_csplus ./migrate_csplus/target/release/migrate_csplus + COPY index.ts . COPY lib ./lib ENV PRODUCTION=true @@ -11,4 +22,4 @@ RUN deno cache index.ts EXPOSE 3000 -CMD ["deno", "run", "--allow-env", "--allow-read", "--allow-write", "--allow-net", "index.ts"] +CMD ["deno", "run", "--allow-env", "--allow-read", "--allow-write", "--allow-net", "--allow-run", "index.ts"] diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index af007f400dece2d2b7ff69f674548fc7030c5b86..65095deb72d26992a653544662acdf99e6662b7e 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -13,3 +13,4 @@ services: DB_USER: ${DB_USER} DB_PASSWORD: ${DB_PASSWORD} DB_DATABASE: ${DB_DATABASE} + IMPORT_CSPLUS_DATA: ${IMPORT_CSPLUS_DATA} diff --git a/index.ts b/index.ts index 7ed6ee75496a78bb55e940e989d96d9029c3833f..e547b495f1c9eb99191687950e48845bfd203373 100644 --- a/index.ts +++ b/index.ts @@ -2,8 +2,9 @@ import { Application, Context, Router } from "https://deno.land/x/oak@v12.6.1/mo import { Client } from "https://deno.land/x/postgres@v0.17.0/mod.ts"; import { load } from "https://deno.land/std@0.209.0/dotenv/mod.ts"; import { updateProfile } from "./lib/update_profile.ts"; +import { isProfilesTableEmpty, runCsplusImport, waitForTableCreation } from "./lib/utils.ts"; -let dbUser, dbDatabase, dbPassword, dbHostname; +let dbUser, dbDatabase, dbPassword, dbHostname, importCsplusData; const dbPort = 5432; const isProduction = Deno.env.get("PRODUCTION") === "true"; @@ -12,12 +13,14 @@ if (isProduction) { dbUser = Deno.env.get("DB_USER"); dbDatabase = Deno.env.get("DB_DATABASE"); dbPassword = Deno.env.get("DB_PASSWORD"); + importCsplusData = Deno.env.get("IMPORT_CSPLUS_DATA") === "true"; dbHostname = "postgres-datapod"; } else { const env = await load(); dbUser = env["DB_USER"]; dbDatabase = env["DB_DATABASE"]; dbPassword = env["DB_PASSWORD"]; + importCsplusData = env["IMPORT_CSPLUS_DATA"] === "true"; dbHostname = "localhost"; } @@ -33,10 +36,20 @@ await client.connect() const app = new Application(); const router = new Router(); +// Wait for table creation before continue +await waitForTableCreation(client, 'public.profiles'); + +// Import Cs+ data +const profilesEmpty = await isProfilesTableEmpty(client); +if (profilesEmpty && importCsplusData) { + await runCsplusImport(isProduction) +} + // Manage routes router.post("/update-profile-data", async (ctx: Context) => await updateProfile(ctx, client)); app.use(router.routes()); app.use(router.allowedMethods()); +console.log("\nDatapod is started") await app.listen({ port: 3000 }); diff --git a/lib/utils.ts b/lib/utils.ts index 8930e845bf5b1c44fdbffea630e516897ab0702d..eb94d95dac8571f14b7257312fdfb9c07842b45e 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -1,3 +1,5 @@ +import { Client } from "https://deno.land/x/postgres@v0.17.0/client.ts"; + export function convertBase64ToBytea(base64String: string): Uint8Array { // Remove the MIME type prefix from the base64 string, if present const base64Data = base64String.split(',')[1] || base64String; @@ -13,3 +15,56 @@ export function convertBase64ToBytea(base64String: string): Uint8Array { return bytes; } + +export async function runCsplusImport(isProduction: boolean) { + const command = new Deno.Command("./migrate_csplus/target/release/migrate_csplus", { + env: { "PRODUCTION": isProduction.toString() }, + stdout: "piped", + stderr: "piped", + }); + + const process = command.spawn(); + + process.stdout.pipeTo(Deno.stdout.writable, { preventClose: true, preventCancel: true, preventAbort: true}); + process.stderr.pipeTo(Deno.stderr.writable, { preventClose: true, preventCancel: true, preventAbort: true}); + + await process.status; + console.log("End of Cs+ data import") +} + +export async function isProfilesTableEmpty(client: Client): Promise<boolean> { + const result = await client.queryObject<{ count: bigint }>('SELECT COUNT(*) FROM public.profiles'); + return result.rows[0].count === 0n; +} + +interface TableCheckResult { + to_regclass: string | null; +} + +async function checkTableExists(client: Client, tableName: string): Promise<boolean> { + try { + const result = await client.queryObject<TableCheckResult>(`SELECT to_regclass('${tableName}')`); + return result.rows[0].to_regclass !== null; + } catch (error) { + console.error("Error checking table existence:", error); + return false; + } +} +function delay(ms: number): Promise<void> { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +export async function waitForTableCreation(client: Client, tableName: string, maxAttempts = 10) { + let attempts = 0; + while (attempts < maxAttempts) { + if (await checkTableExists(client, tableName)) { + console.log(`Table ${tableName} found.`); + return; + } + + await delay(1000); + attempts++; + } + + throw new Error(`Table ${tableName} not found after ${maxAttempts} try.`); +} diff --git a/load.sh b/load.sh index fd1d8d7c6825b0e7dea66ac16236a5b72a1a6167..5765edc6c68bd1390c400b2516adc3e591863e0d 100755 --- a/load.sh +++ b/load.sh @@ -6,7 +6,7 @@ if [[ $option == "dev" ]]; then echo "Start datapod in dev mode" docker compose -f docker-compose.yml -f docker-compose.prod.yml -f docker-compose.override.yml down -v docker compose up -d - deno run --allow-env --allow-read --allow-write --allow-net --watch index.ts + deno run --allow-env --allow-read --allow-write --allow-net --allow-run --watch index.ts else echo "Start datapod in production mode" docker compose -f docker-compose.yml -f docker-compose.prod.yml -f docker-compose.override.yml down diff --git a/scrap_csplus/Cargo.lock b/migrate_csplus/Cargo.lock similarity index 99% rename from scrap_csplus/Cargo.lock rename to migrate_csplus/Cargo.lock index f4005a37e02c57d6ad0ed9ac6cbb175265ed6a80..49c8e743934b2ce83abe30b355da73bdfb28ac09 100644 --- a/scrap_csplus/Cargo.lock +++ b/migrate_csplus/Cargo.lock @@ -1420,6 +1420,21 @@ dependencies = [ "zeroize", ] +[[package]] +name = "migrate_csplus" +version = "0.1.0" +dependencies = [ + "base64 0.21.5", + "bs58", + "dotenv", + "reqwest", + "serde", + "serde_json", + "sp-core", + "tokio", + "tokio-postgres", +] + [[package]] name = "mime" version = "0.3.17" @@ -2102,21 +2117,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "scrap_csplus" -version = "0.1.0" -dependencies = [ - "base64 0.21.5", - "bs58", - "dotenv", - "reqwest", - "serde", - "serde_json", - "sp-core", - "tokio", - "tokio-postgres", -] - [[package]] name = "secp256k1" version = "0.28.0" diff --git a/scrap_csplus/Cargo.toml b/migrate_csplus/Cargo.toml similarity index 95% rename from scrap_csplus/Cargo.toml rename to migrate_csplus/Cargo.toml index 62830fee82cbf3f7bce26c6e888e6dd90f6c6389..5c612e71a98b69fc5d316f46045140a4dcf299a6 100644 --- a/scrap_csplus/Cargo.toml +++ b/migrate_csplus/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "scrap_csplus" +name = "migrate_csplus" version = "0.1.0" edition = "2018" diff --git a/scrap_csplus/src/import_only.rs b/migrate_csplus/src/import_only.rs similarity index 100% rename from scrap_csplus/src/import_only.rs rename to migrate_csplus/src/import_only.rs diff --git a/scrap_csplus/src/main.rs b/migrate_csplus/src/main.rs similarity index 99% rename from scrap_csplus/src/main.rs rename to migrate_csplus/src/main.rs index 644c8e9befc47ab9e2ac44ff774f25139436862c..9b4c690f39f345d46aea9d585201e94b79a7d16e 100644 --- a/scrap_csplus/src/main.rs +++ b/migrate_csplus/src/main.rs @@ -250,6 +250,7 @@ async fn fetch_profiles() -> Result<(), Box<dyn Error>> { #[tokio::main] async fn main() { + println!("Starting Cs+ data migration..."); if let Err(e) = fetch_profiles().await { eprintln!("Error: {}", e); }