[{"data":1,"prerenderedAt":3333},["ShallowReactive",2],{"post:\u002F2026\u002F02\u002F25\u002Fnode-js-data-warehouse-streaming-etl-rows-without-loading-them-all-in-memory\u002F":3},{"post":4,"newer":3287,"older":3296,"related":3305,"series":3331},{"id":5,"title":6,"body":7,"canonical":3267,"categories":3268,"date":3271,"description":3272,"extension":3273,"featured":3274,"hero":3275,"image":3275,"meta":3276,"navigation":503,"path":3277,"readingTime":247,"seo":3278,"series":3275,"seriesOrder":3275,"sites":3279,"source":3275,"stem":3280,"tags":3281,"updated":3275,"url":3285,"__hash__":3286},"blog\u002Fblog\u002F2026\u002F02\u002F25\u002Fnode-js-data-warehouse-streaming-etl-rows-without-loading-them-all-in-memory.md","Node.js: Data Warehouse – Streaming ETL Rows Without Buffering Them",{"type":8,"value":9,"toc":3260},"minimark",[10,23,30,119,124,313,317,333,396,399,407,414,418,3020,3024,3204,3208,3256],[11,12,13,14,18,19,22],"p",{},"The failure mode I keep seeing in hand-rolled ETL jobs is the same one every time: ",[15,16,17],"code",{},"SELECT * FROM source_table",", load the whole result set into an array, transform it in memory, then loop over the array and insert rows one at a time into the destination. It works fine in development against a few thousand rows and falls over in production, either by exhausting the process's memory or by taking so long on row-by-row inserts that the job never finishes inside its window. Node's streams solve both problems if you actually use them as streams rather than as a ",[15,20,21],{},".on('data')"," handler that pushes into an array.",[11,24,25,26,29],{},"This script pulls rows from a source Postgres table through a cursor, transforms them one at a time, and bulk-loads them into a destination Postgres table (a staging table in a warehouse, say) with ",[15,27,28],{},"COPY ... FROM STDIN",". Three details make it correct rather than just fast:",[31,32,33,56,103],"ul",{},[34,35,36,40,41,44,45,48,49,52,53,55],"li",{},[37,38,39],"strong",{},"Backpressure end to end."," ",[15,42,43],{},"pipeline()"," from ",[15,46,47],{},"node:stream\u002Fpromises"," connects the cursor, the transform, and the ",[15,50,51],{},"COPY"," stream. When ",[15,54,51],{}," falls behind, the cursor stops fetching.",[34,57,58,61,62,65,66,69,70,73,74,77,78,81,82,85,86,89,90,93,94,96,97,99,100,102],{},[37,59,60],{},"Raw text in, raw text out."," By default ",[15,63,64],{},"pg"," turns ",[15,67,68],{},"timestamptz"," into a JavaScript ",[15,71,72],{},"Date"," and ",[15,75,76],{},"json","\u002F",[15,79,80],{},"jsonb"," into an object. Written back with ",[15,83,84],{},"String(value)",", a date becomes ",[15,87,88],{},"Tue Feb 24 2026 ..."," and an object becomes ",[15,91,92],{},"[object Object]",". That is an easy bug to ship, because nothing fails until the load hits a ",[15,95,68],{}," or ",[15,98,80],{}," column. This version tells the cursor to skip type parsing, so every value stays the text Postgres sent, which ",[15,101,51],{}," reads back as-is.",[34,104,105,40,108,111,112,115,116,118],{},[37,106,107],{},"An explicit column list.",[15,109,110],{},"COPY table FROM STDIN"," without a column list assumes the table's physical column order. The script reads the destination's writable (non-generated) columns from ",[15,113,114],{},"information_schema.columns",", names them in the ",[15,117,51],{},", and fails on the first row if the transform doesn't produce every one of them.",[120,121,123],"h2",{"id":122},"requirements","Requirements",[31,125,126,129,150,173],{},[34,127,128],{},"A supported Node.js LTS release (22 or later).",[34,130,131,133,134,137,138,141,142,145,146,149],{},[15,132,64],{},", ",[15,135,136],{},"pg-query-stream",", and ",[15,139,140],{},"pg-copy-streams"," 6 or later (",[15,143,144],{},"npm install pg pg-query-stream pg-copy-streams","). Both stream modules only work with the pure JavaScript client, not ",[15,147,148],{},"pg-native",".",[34,151,152,153,156,157,160,161,164,165,168,169,172],{},"A source role with ",[15,154,155],{},"SELECT"," on the source table, and a destination role with ",[15,158,159],{},"INSERT"," (plus ",[15,162,163],{},"TRUNCATE"," if you use ",[15,166,167],{},"--truncate",") on the destination table. Each role also needs ",[15,170,171],{},"USAGE"," on the table's schema if it doesn't own it.",[34,174,175,176,179,180],{},"The destination table created ahead of time. For the default transform below it needs every source column plus a ",[15,177,178],{},"loaded_at timestamptz"," column:",[181,182,187],"pre",{"className":183,"code":184,"language":185,"meta":186,"style":186},"language-sql shiki shiki-themes github-dark","CREATE TABLE staging.stg_orders (\n    id          bigint PRIMARY KEY,\n    customer_id bigint,\n    status      text,\n    total       numeric(12,2),\n    metadata    jsonb,\n    created_at  timestamptz,\n    updated_at  timestamptz,\n    loaded_at   timestamptz NOT NULL\n);\n","sql","",[15,188,189,209,224,234,245,270,276,286,296,307],{"__ignoreMap":186},[190,191,194,198,201,205],"span",{"class":192,"line":193},"line",1,[190,195,197],{"class":196},"snl16","CREATE",[190,199,200],{"class":196}," TABLE",[190,202,204],{"class":203},"svObZ"," staging",[190,206,208],{"class":207},"s95oV",".stg_orders (\n",[190,210,212,215,218,221],{"class":192,"line":211},2,[190,213,214],{"class":207},"    id          ",[190,216,217],{"class":196},"bigint",[190,219,220],{"class":196}," PRIMARY KEY",[190,222,223],{"class":207},",\n",[190,225,227,230,232],{"class":192,"line":226},3,[190,228,229],{"class":207},"    customer_id ",[190,231,217],{"class":196},[190,233,223],{"class":207},[190,235,237,240,243],{"class":192,"line":236},4,[190,238,239],{"class":196},"    status",[190,241,242],{"class":196},"      text",[190,244,223],{"class":207},[190,246,248,251,254,257,261,264,267],{"class":192,"line":247},5,[190,249,250],{"class":207},"    total       ",[190,252,253],{"class":196},"numeric",[190,255,256],{"class":207},"(",[190,258,260],{"class":259},"sDLfK","12",[190,262,263],{"class":207},",",[190,265,266],{"class":259},"2",[190,268,269],{"class":207},"),\n",[190,271,273],{"class":192,"line":272},6,[190,274,275],{"class":207},"    metadata    jsonb,\n",[190,277,279,282,284],{"class":192,"line":278},7,[190,280,281],{"class":207},"    created_at  ",[190,283,68],{"class":196},[190,285,223],{"class":207},[190,287,289,292,294],{"class":192,"line":288},8,[190,290,291],{"class":207},"    updated_at  ",[190,293,68],{"class":196},[190,295,223],{"class":207},[190,297,299,302,304],{"class":192,"line":298},9,[190,300,301],{"class":207},"    loaded_at   ",[190,303,68],{"class":196},[190,305,306],{"class":196}," NOT NULL\n",[190,308,310],{"class":192,"line":309},10,[190,311,312],{"class":207},");\n",[120,314,316],{"id":315},"usage","Usage",[11,318,319,320,322,323,325,326,328,329,332],{},"Set the connection strings and run the job. ",[15,321,167],{}," empties the destination in the same transaction as the ",[15,324,51],{},", so readers never see a half-loaded table. ",[15,327,163],{}," takes an ",[15,330,331],{},"ACCESS EXCLUSIVE"," lock, so readers wait until the commit, and it isn't MVCC-safe: a transaction whose snapshot predates the commit and hadn't yet touched the table sees it empty. If that matters, load a staging table and swap it in.",[181,334,338],{"className":335,"code":336,"language":337,"meta":186,"style":186},"language-bash shiki shiki-themes github-dark","export SOURCE_DATABASE_URL=\"postgres:\u002F\u002Fetl_reader:\u003Cpassword>@\u003Csource-host>:5432\u002Fapp\"\nexport DEST_DATABASE_URL=\"postgres:\u002F\u002Fetl_writer:\u003Cpassword>@\u003Cwarehouse-host>:5432\u002Fwarehouse\"\nnode stream_etl.js --source-table public.orders --dest-table staging.stg_orders --truncate --log-every 50000\n","bash",[15,339,340,355,367],{"__ignoreMap":186},[190,341,342,345,348,351],{"class":192,"line":193},[190,343,344],{"class":196},"export",[190,346,347],{"class":207}," SOURCE_DATABASE_URL",[190,349,350],{"class":196},"=",[190,352,354],{"class":353},"sU2Wk","\"postgres:\u002F\u002Fetl_reader:\u003Cpassword>@\u003Csource-host>:5432\u002Fapp\"\n",[190,356,357,359,362,364],{"class":192,"line":211},[190,358,344],{"class":196},[190,360,361],{"class":207}," DEST_DATABASE_URL",[190,363,350],{"class":196},[190,365,366],{"class":353},"\"postgres:\u002F\u002Fetl_writer:\u003Cpassword>@\u003Cwarehouse-host>:5432\u002Fwarehouse\"\n",[190,368,369,372,375,378,381,384,387,390,393],{"class":192,"line":226},[190,370,371],{"class":203},"node",[190,373,374],{"class":353}," stream_etl.js",[190,376,377],{"class":259}," --source-table",[190,379,380],{"class":353}," public.orders",[190,382,383],{"class":259}," --dest-table",[190,385,386],{"class":353}," staging.stg_orders",[190,388,389],{"class":259}," --truncate",[190,391,392],{"class":259}," --log-every",[190,394,395],{"class":259}," 50000\n",[11,397,398],{},"Sample output:",[181,400,405],{"className":401,"code":403,"language":404,"meta":186},[402],"language-text","Streaming public.orders -> staging.stg_orders (8 columns)\nProcessed 50,000 rows (heap 21 MB)\nProcessed 100,000 rows (heap 23 MB)\nDone: 137,412 rows in 38.6s, committed\n","text",[15,406,403],{"__ignoreMap":186},[11,408,409,410,413],{},"The heap figure is ",[15,411,412],{},"process.memoryUsage().heapUsed"," at that moment. What matters is that it stays roughly flat as the row count climbs; if it grows with the row count, something in the transform is holding onto rows.",[120,415,417],{"id":416},"script","Script",[181,419,423],{"className":420,"code":421,"language":422,"meta":186,"style":186},"language-javascript shiki shiki-themes github-dark","\u002F**\n * stream_etl.js\n *\n * Streams rows from a source Postgres table through a cursor, applies a\n * per-row transform, and bulk-loads the result into a destination table with\n * COPY ... FROM STDIN, without buffering the result set in memory.\n *\n * Reads:  SOURCE_DATABASE_URL, --source-table (SELECT *).\n *         DEST_DATABASE_URL, information_schema.columns for --dest-table.\n * Writes: --dest-table via COPY, inside one transaction (optionally after TRUNCATE).\n *\n * Usage: node stream_etl.js --source-table \u003Cschema.table> --dest-table \u003Cschema.table>\n *                           [--truncate] [--batch-size 1000] [--log-every 50000]\n *\u002F\n\n\"use strict\";\n\nconst { Pool } = require(\"pg\");\nconst QueryStream = require(\"pg-query-stream\");\nconst { from: copyFrom } = require(\"pg-copy-streams\");\nconst { Transform } = require(\"node:stream\");\nconst { pipeline } = require(\"node:stream\u002Fpromises\");\n\n\u002F\u002F Skip pg's type parsing: every non-null value arrives as the text Postgres sent.\nconst RAW_TEXT = { getTypeParser: () => (value) => value };\n\nfunction parseArgs(argv) {\n    const args = { truncate: false, \"batch-size\": \"1000\", \"log-every\": \"50000\" };\n    for (let i = 0; i \u003C argv.length; i += 1) {\n        const token = argv[i];\n        if (!token.startsWith(\"--\")) {\n            throw new Error(`Unexpected argument: ${token}`);\n        }\n        const key = token.slice(2);\n        if (key === \"truncate\") {\n            args.truncate = true;\n        } else {\n            args[key] = argv[i + 1];\n            i += 1;\n        }\n    }\n    if (!args[\"source-table\"] || !args[\"dest-table\"]) {\n        throw new Error(\"Usage: node stream_etl.js --source-table \u003Cschema.table> --dest-table \u003Cschema.table> [--truncate]\");\n    }\n    const batchSize = Number(args[\"batch-size\"]);\n    const logEvery = Number(args[\"log-every\"]);\n    if (!Number.isInteger(batchSize) || batchSize \u003C 1 || !Number.isInteger(logEvery) || logEvery \u003C 1) {\n        throw new Error(\"--batch-size and --log-every must be positive integers\");\n    }\n    return {\n        sourceTable: args[\"source-table\"],\n        destTable: args[\"dest-table\"],\n        truncate: args.truncate,\n        batchSize,\n        logEvery,\n    };\n}\n\nfunction splitName(name) {\n    const parts = name.split(\".\");\n    return parts.length === 2 ? parts : [\"public\", parts[0]];\n}\n\nfunction quoteName(client, name) {\n    return splitName(name).map((part) => client.escapeIdentifier(part)).join(\".\");\n}\n\nasync function destinationColumns(client, name) {\n    const [schema, table] = splitName(name);\n    const { rows } = await client.query(\n        `SELECT column_name\n           FROM information_schema.columns\n          WHERE table_schema = $1 AND table_name = $2\n            AND is_generated = 'NEVER'\n          ORDER BY ordinal_position`,\n        [schema, table],\n    );\n    if (rows.length === 0) {\n        throw new Error(`Destination table ${name} not found or not visible to this role`);\n    }\n    return rows.map((row) => row.column_name);\n}\n\n\u002F\u002F One row in, one row out. Values are Postgres text (or null). Extend this for\n\u002F\u002F renames, filtering, or derived columns; never hold a reference to past rows.\nfunction transformRow(row, loadedAt) {\n    return { ...row, loaded_at: loadedAt };\n}\n\n\u002F\u002F COPY text format: tab-delimited, \\N for null, and backslash, newline,\n\u002F\u002F carriage return, and the delimiter escaped with a backslash.\nfunction toCopyField(value) {\n    if (value === null || value === undefined) {\n        return \"\\\\N\";\n    }\n    return String(value)\n        .replace(\u002F\\\\\u002Fg, \"\\\\\\\\\")\n        .replace(\u002F\\t\u002Fg, \"\\\\t\")\n        .replace(\u002F\\n\u002Fg, \"\\\\n\")\n        .replace(\u002F\\r\u002Fg, \"\\\\r\");\n}\n\nasync function run() {\n    const { sourceTable, destTable, truncate, batchSize, logEvery } = parseArgs(process.argv.slice(2));\n\n    const sourcePool = new Pool({ connectionString: process.env.SOURCE_DATABASE_URL, max: 1 });\n    const destPool = new Pool({ connectionString: process.env.DEST_DATABASE_URL, max: 1 });\n    const sourceClient = await sourcePool.connect();\n    const destClient = await destPool.connect();\n\n    const startedAt = Date.now();\n    const loadedAt = new Date().toISOString();\n    let rowCount = 0;\n\n    try {\n        const columns = await destinationColumns(destClient, destTable);\n        const columnList = columns.map((c) => destClient.escapeIdentifier(c)).join(\", \");\n        console.log(`Streaming ${sourceTable} -> ${destTable} (${columns.length} columns)`);\n\n        await destClient.query(\"BEGIN\");\n        if (truncate) {\n            await destClient.query(`TRUNCATE ${quoteName(destClient, destTable)}`);\n        }\n\n        const source = sourceClient.query(\n            new QueryStream(`SELECT * FROM ${quoteName(sourceClient, sourceTable)}`, [], {\n                batchSize,\n                types: RAW_TEXT,\n            }),\n        );\n\n        const toCopyLines = new Transform({\n            writableObjectMode: true,\n            readableObjectMode: false,\n            transform(row, _encoding, callback) {\n                const out = transformRow(row, loadedAt);\n                if (rowCount === 0) {\n                    const missing = columns.filter((c) => !(c in out));\n                    if (missing.length > 0) {\n                        callback(new Error(`Transform does not produce destination column(s): ${missing.join(\", \")}`));\n                        return;\n                    }\n                }\n                rowCount += 1;\n                if (rowCount % logEvery === 0) {\n                    const heapMb = Math.round(process.memoryUsage().heapUsed \u002F 1024 \u002F 1024);\n                    console.log(`Processed ${rowCount.toLocaleString()} rows (heap ${heapMb} MB)`);\n                }\n                callback(null, columns.map((c) => toCopyField(out[c])).join(\"\\t\") + \"\\n\");\n            },\n        });\n\n        const sink = destClient.query(\n            copyFrom(`COPY ${quoteName(destClient, destTable)} (${columnList}) FROM STDIN`),\n        );\n\n        \u002F\u002F Propagates backpressure and destroys every stream if any one fails;\n        \u002F\u002F destroying the COPY stream sends CopyFail so the server aborts the COPY.\n        await pipeline(source, toCopyLines, sink);\n        await destClient.query(\"COMMIT\");\n\n        const seconds = ((Date.now() - startedAt) \u002F 1000).toFixed(1);\n        console.log(`Done: ${rowCount.toLocaleString()} rows in ${seconds}s, committed`);\n    } catch (error) {\n        await destClient.query(\"ROLLBACK\").catch(() => {});\n        throw error;\n    } finally {\n        sourceClient.release();\n        destClient.release();\n        await Promise.all([sourcePool.end(), destPool.end()]);\n    }\n}\n\nrun().catch((error) => {\n    console.error(\"ETL job failed:\", error.message);\n    process.exitCode = 1;\n});\n","javascript",[15,424,425,431,436,441,446,451,456,460,465,470,475,480,486,492,498,505,514,519,546,566,596,619,642,647,653,688,693,710,750,791,805,830,855,861,883,899,912,924,943,955,960,966,999,1016,1021,1042,1060,1110,1126,1131,1139,1150,1160,1166,1172,1178,1184,1190,1195,1210,1233,1273,1278,1283,1302,1343,1348,1353,1375,1399,1424,1430,1436,1442,1448,1456,1462,1468,1484,1505,1510,1532,1537,1542,1548,1554,1573,1586,1591,1596,1602,1608,1622,1647,1664,1669,1680,1714,1743,1772,1801,1806,1811,1824,1872,1877,1907,1932,1953,1972,1977,1995,2018,2033,2038,2046,2063,2104,2140,2145,2162,2170,2204,2209,2214,2231,2262,2268,2279,2285,2291,2296,2314,2325,2335,2357,2373,2388,2423,2441,2476,2484,2490,2496,2508,2526,2563,2598,2603,2654,2660,2666,2671,2687,2720,2725,2730,2736,2742,2753,2769,2774,2815,2846,2858,2884,2892,2902,2913,2923,2950,2955,2960,2965,2986,3002,3014],{"__ignoreMap":186},[190,426,427],{"class":192,"line":193},[190,428,430],{"class":429},"sAwPA","\u002F**\n",[190,432,433],{"class":192,"line":211},[190,434,435],{"class":429}," * stream_etl.js\n",[190,437,438],{"class":192,"line":226},[190,439,440],{"class":429}," *\n",[190,442,443],{"class":192,"line":236},[190,444,445],{"class":429}," * Streams rows from a source Postgres table through a cursor, applies a\n",[190,447,448],{"class":192,"line":247},[190,449,450],{"class":429}," * per-row transform, and bulk-loads the result into a destination table with\n",[190,452,453],{"class":192,"line":272},[190,454,455],{"class":429}," * COPY ... FROM STDIN, without buffering the result set in memory.\n",[190,457,458],{"class":192,"line":278},[190,459,440],{"class":429},[190,461,462],{"class":192,"line":288},[190,463,464],{"class":429}," * Reads:  SOURCE_DATABASE_URL, --source-table (SELECT *).\n",[190,466,467],{"class":192,"line":298},[190,468,469],{"class":429}," *         DEST_DATABASE_URL, information_schema.columns for --dest-table.\n",[190,471,472],{"class":192,"line":309},[190,473,474],{"class":429}," * Writes: --dest-table via COPY, inside one transaction (optionally after TRUNCATE).\n",[190,476,478],{"class":192,"line":477},11,[190,479,440],{"class":429},[190,481,483],{"class":192,"line":482},12,[190,484,485],{"class":429}," * Usage: node stream_etl.js --source-table \u003Cschema.table> --dest-table \u003Cschema.table>\n",[190,487,489],{"class":192,"line":488},13,[190,490,491],{"class":429}," *                           [--truncate] [--batch-size 1000] [--log-every 50000]\n",[190,493,495],{"class":192,"line":494},14,[190,496,497],{"class":429}," *\u002F\n",[190,499,501],{"class":192,"line":500},15,[190,502,504],{"emptyLinePlaceholder":503},true,"\n",[190,506,508,511],{"class":192,"line":507},16,[190,509,510],{"class":353},"\"use strict\"",[190,512,513],{"class":207},";\n",[190,515,517],{"class":192,"line":516},17,[190,518,504],{"emptyLinePlaceholder":503},[190,520,522,525,528,531,534,536,539,541,544],{"class":192,"line":521},18,[190,523,524],{"class":196},"const",[190,526,527],{"class":207}," { ",[190,529,530],{"class":259},"Pool",[190,532,533],{"class":207}," } ",[190,535,350],{"class":196},[190,537,538],{"class":203}," require",[190,540,256],{"class":207},[190,542,543],{"class":353},"\"pg\"",[190,545,312],{"class":207},[190,547,549,551,554,557,559,561,564],{"class":192,"line":548},19,[190,550,524],{"class":196},[190,552,553],{"class":259}," QueryStream",[190,555,556],{"class":196}," =",[190,558,538],{"class":203},[190,560,256],{"class":207},[190,562,563],{"class":353},"\"pg-query-stream\"",[190,565,312],{"class":207},[190,567,569,571,573,577,580,583,585,587,589,591,594],{"class":192,"line":568},20,[190,570,524],{"class":196},[190,572,527],{"class":207},[190,574,576],{"class":575},"s9osk","from",[190,578,579],{"class":207},": ",[190,581,582],{"class":259},"copyFrom",[190,584,533],{"class":207},[190,586,350],{"class":196},[190,588,538],{"class":203},[190,590,256],{"class":207},[190,592,593],{"class":353},"\"pg-copy-streams\"",[190,595,312],{"class":207},[190,597,599,601,603,606,608,610,612,614,617],{"class":192,"line":598},21,[190,600,524],{"class":196},[190,602,527],{"class":207},[190,604,605],{"class":259},"Transform",[190,607,533],{"class":207},[190,609,350],{"class":196},[190,611,538],{"class":203},[190,613,256],{"class":207},[190,615,616],{"class":353},"\"node:stream\"",[190,618,312],{"class":207},[190,620,622,624,626,629,631,633,635,637,640],{"class":192,"line":621},22,[190,623,524],{"class":196},[190,625,527],{"class":207},[190,627,628],{"class":259},"pipeline",[190,630,533],{"class":207},[190,632,350],{"class":196},[190,634,538],{"class":203},[190,636,256],{"class":207},[190,638,639],{"class":353},"\"node:stream\u002Fpromises\"",[190,641,312],{"class":207},[190,643,645],{"class":192,"line":644},23,[190,646,504],{"emptyLinePlaceholder":503},[190,648,650],{"class":192,"line":649},24,[190,651,652],{"class":429},"\u002F\u002F Skip pg's type parsing: every non-null value arrives as the text Postgres sent.\n",[190,654,656,658,661,663,665,668,671,674,677,680,683,685],{"class":192,"line":655},25,[190,657,524],{"class":196},[190,659,660],{"class":259}," RAW_TEXT",[190,662,556],{"class":196},[190,664,527],{"class":207},[190,666,667],{"class":203},"getTypeParser",[190,669,670],{"class":207},": () ",[190,672,673],{"class":196},"=>",[190,675,676],{"class":207}," (",[190,678,679],{"class":575},"value",[190,681,682],{"class":207},") ",[190,684,673],{"class":196},[190,686,687],{"class":207}," value };\n",[190,689,691],{"class":192,"line":690},26,[190,692,504],{"emptyLinePlaceholder":503},[190,694,696,699,702,704,707],{"class":192,"line":695},27,[190,697,698],{"class":196},"function",[190,700,701],{"class":203}," parseArgs",[190,703,256],{"class":207},[190,705,706],{"class":575},"argv",[190,708,709],{"class":207},") {\n",[190,711,713,716,719,721,724,727,729,732,734,737,739,742,744,747],{"class":192,"line":712},28,[190,714,715],{"class":196},"    const",[190,717,718],{"class":259}," args",[190,720,556],{"class":196},[190,722,723],{"class":207}," { truncate: ",[190,725,726],{"class":259},"false",[190,728,133],{"class":207},[190,730,731],{"class":353},"\"batch-size\"",[190,733,579],{"class":207},[190,735,736],{"class":353},"\"1000\"",[190,738,133],{"class":207},[190,740,741],{"class":353},"\"log-every\"",[190,743,579],{"class":207},[190,745,746],{"class":353},"\"50000\"",[190,748,749],{"class":207}," };\n",[190,751,753,756,758,761,764,766,769,772,775,778,781,783,786,789],{"class":192,"line":752},29,[190,754,755],{"class":196},"    for",[190,757,676],{"class":207},[190,759,760],{"class":196},"let",[190,762,763],{"class":207}," i ",[190,765,350],{"class":196},[190,767,768],{"class":259}," 0",[190,770,771],{"class":207},"; i ",[190,773,774],{"class":196},"\u003C",[190,776,777],{"class":207}," argv.",[190,779,780],{"class":259},"length",[190,782,771],{"class":207},[190,784,785],{"class":196},"+=",[190,787,788],{"class":259}," 1",[190,790,709],{"class":207},[190,792,794,797,800,802],{"class":192,"line":793},30,[190,795,796],{"class":196},"        const",[190,798,799],{"class":259}," token",[190,801,556],{"class":196},[190,803,804],{"class":207}," argv[i];\n",[190,806,808,811,813,816,819,822,824,827],{"class":192,"line":807},31,[190,809,810],{"class":196},"        if",[190,812,676],{"class":207},[190,814,815],{"class":196},"!",[190,817,818],{"class":207},"token.",[190,820,821],{"class":203},"startsWith",[190,823,256],{"class":207},[190,825,826],{"class":353},"\"--\"",[190,828,829],{"class":207},")) {\n",[190,831,833,836,839,842,844,847,850,853],{"class":192,"line":832},32,[190,834,835],{"class":196},"            throw",[190,837,838],{"class":196}," new",[190,840,841],{"class":203}," Error",[190,843,256],{"class":207},[190,845,846],{"class":353},"`Unexpected argument: ${",[190,848,849],{"class":207},"token",[190,851,852],{"class":353},"}`",[190,854,312],{"class":207},[190,856,858],{"class":192,"line":857},33,[190,859,860],{"class":207},"        }\n",[190,862,864,866,869,871,874,877,879,881],{"class":192,"line":863},34,[190,865,796],{"class":196},[190,867,868],{"class":259}," key",[190,870,556],{"class":196},[190,872,873],{"class":207}," token.",[190,875,876],{"class":203},"slice",[190,878,256],{"class":207},[190,880,266],{"class":259},[190,882,312],{"class":207},[190,884,886,888,891,894,897],{"class":192,"line":885},35,[190,887,810],{"class":196},[190,889,890],{"class":207}," (key ",[190,892,893],{"class":196},"===",[190,895,896],{"class":353}," \"truncate\"",[190,898,709],{"class":207},[190,900,902,905,907,910],{"class":192,"line":901},36,[190,903,904],{"class":207},"            args.truncate ",[190,906,350],{"class":196},[190,908,909],{"class":259}," true",[190,911,513],{"class":207},[190,913,915,918,921],{"class":192,"line":914},37,[190,916,917],{"class":207},"        } ",[190,919,920],{"class":196},"else",[190,922,923],{"class":207}," {\n",[190,925,927,930,932,935,938,940],{"class":192,"line":926},38,[190,928,929],{"class":207},"            args[key] ",[190,931,350],{"class":196},[190,933,934],{"class":207}," argv[i ",[190,936,937],{"class":196},"+",[190,939,788],{"class":259},[190,941,942],{"class":207},"];\n",[190,944,946,949,951,953],{"class":192,"line":945},39,[190,947,948],{"class":207},"            i ",[190,950,785],{"class":196},[190,952,788],{"class":259},[190,954,513],{"class":207},[190,956,958],{"class":192,"line":957},40,[190,959,860],{"class":207},[190,961,963],{"class":192,"line":962},41,[190,964,965],{"class":207},"    }\n",[190,967,969,972,974,976,979,982,985,988,991,993,996],{"class":192,"line":968},42,[190,970,971],{"class":196},"    if",[190,973,676],{"class":207},[190,975,815],{"class":196},[190,977,978],{"class":207},"args[",[190,980,981],{"class":353},"\"source-table\"",[190,983,984],{"class":207},"] ",[190,986,987],{"class":196},"||",[190,989,990],{"class":196}," !",[190,992,978],{"class":207},[190,994,995],{"class":353},"\"dest-table\"",[190,997,998],{"class":207},"]) {\n",[190,1000,1002,1005,1007,1009,1011,1014],{"class":192,"line":1001},43,[190,1003,1004],{"class":196},"        throw",[190,1006,838],{"class":196},[190,1008,841],{"class":203},[190,1010,256],{"class":207},[190,1012,1013],{"class":353},"\"Usage: node stream_etl.js --source-table \u003Cschema.table> --dest-table \u003Cschema.table> [--truncate]\"",[190,1015,312],{"class":207},[190,1017,1019],{"class":192,"line":1018},44,[190,1020,965],{"class":207},[190,1022,1024,1026,1029,1031,1034,1037,1039],{"class":192,"line":1023},45,[190,1025,715],{"class":196},[190,1027,1028],{"class":259}," batchSize",[190,1030,556],{"class":196},[190,1032,1033],{"class":203}," Number",[190,1035,1036],{"class":207},"(args[",[190,1038,731],{"class":353},[190,1040,1041],{"class":207},"]);\n",[190,1043,1045,1047,1050,1052,1054,1056,1058],{"class":192,"line":1044},46,[190,1046,715],{"class":196},[190,1048,1049],{"class":259}," logEvery",[190,1051,556],{"class":196},[190,1053,1033],{"class":203},[190,1055,1036],{"class":207},[190,1057,741],{"class":353},[190,1059,1041],{"class":207},[190,1061,1063,1065,1067,1069,1072,1075,1078,1080,1083,1085,1087,1090,1092,1094,1096,1099,1101,1104,1106,1108],{"class":192,"line":1062},47,[190,1064,971],{"class":196},[190,1066,676],{"class":207},[190,1068,815],{"class":196},[190,1070,1071],{"class":207},"Number.",[190,1073,1074],{"class":203},"isInteger",[190,1076,1077],{"class":207},"(batchSize) ",[190,1079,987],{"class":196},[190,1081,1082],{"class":207}," batchSize ",[190,1084,774],{"class":196},[190,1086,788],{"class":259},[190,1088,1089],{"class":196}," ||",[190,1091,990],{"class":196},[190,1093,1071],{"class":207},[190,1095,1074],{"class":203},[190,1097,1098],{"class":207},"(logEvery) ",[190,1100,987],{"class":196},[190,1102,1103],{"class":207}," logEvery ",[190,1105,774],{"class":196},[190,1107,788],{"class":259},[190,1109,709],{"class":207},[190,1111,1113,1115,1117,1119,1121,1124],{"class":192,"line":1112},48,[190,1114,1004],{"class":196},[190,1116,838],{"class":196},[190,1118,841],{"class":203},[190,1120,256],{"class":207},[190,1122,1123],{"class":353},"\"--batch-size and --log-every must be positive integers\"",[190,1125,312],{"class":207},[190,1127,1129],{"class":192,"line":1128},49,[190,1130,965],{"class":207},[190,1132,1134,1137],{"class":192,"line":1133},50,[190,1135,1136],{"class":196},"    return",[190,1138,923],{"class":207},[190,1140,1142,1145,1147],{"class":192,"line":1141},51,[190,1143,1144],{"class":207},"        sourceTable: args[",[190,1146,981],{"class":353},[190,1148,1149],{"class":207},"],\n",[190,1151,1153,1156,1158],{"class":192,"line":1152},52,[190,1154,1155],{"class":207},"        destTable: args[",[190,1157,995],{"class":353},[190,1159,1149],{"class":207},[190,1161,1163],{"class":192,"line":1162},53,[190,1164,1165],{"class":207},"        truncate: args.truncate,\n",[190,1167,1169],{"class":192,"line":1168},54,[190,1170,1171],{"class":207},"        batchSize,\n",[190,1173,1175],{"class":192,"line":1174},55,[190,1176,1177],{"class":207},"        logEvery,\n",[190,1179,1181],{"class":192,"line":1180},56,[190,1182,1183],{"class":207},"    };\n",[190,1185,1187],{"class":192,"line":1186},57,[190,1188,1189],{"class":207},"}\n",[190,1191,1193],{"class":192,"line":1192},58,[190,1194,504],{"emptyLinePlaceholder":503},[190,1196,1198,1200,1203,1205,1208],{"class":192,"line":1197},59,[190,1199,698],{"class":196},[190,1201,1202],{"class":203}," splitName",[190,1204,256],{"class":207},[190,1206,1207],{"class":575},"name",[190,1209,709],{"class":207},[190,1211,1213,1215,1218,1220,1223,1226,1228,1231],{"class":192,"line":1212},60,[190,1214,715],{"class":196},[190,1216,1217],{"class":259}," parts",[190,1219,556],{"class":196},[190,1221,1222],{"class":207}," name.",[190,1224,1225],{"class":203},"split",[190,1227,256],{"class":207},[190,1229,1230],{"class":353},"\".\"",[190,1232,312],{"class":207},[190,1234,1236,1238,1241,1243,1246,1249,1252,1255,1258,1261,1264,1267,1270],{"class":192,"line":1235},61,[190,1237,1136],{"class":196},[190,1239,1240],{"class":207}," parts.",[190,1242,780],{"class":259},[190,1244,1245],{"class":196}," ===",[190,1247,1248],{"class":259}," 2",[190,1250,1251],{"class":196}," ?",[190,1253,1254],{"class":207}," parts ",[190,1256,1257],{"class":196},":",[190,1259,1260],{"class":207}," [",[190,1262,1263],{"class":353},"\"public\"",[190,1265,1266],{"class":207},", parts[",[190,1268,1269],{"class":259},"0",[190,1271,1272],{"class":207},"]];\n",[190,1274,1276],{"class":192,"line":1275},62,[190,1277,1189],{"class":207},[190,1279,1281],{"class":192,"line":1280},63,[190,1282,504],{"emptyLinePlaceholder":503},[190,1284,1286,1288,1291,1293,1296,1298,1300],{"class":192,"line":1285},64,[190,1287,698],{"class":196},[190,1289,1290],{"class":203}," quoteName",[190,1292,256],{"class":207},[190,1294,1295],{"class":575},"client",[190,1297,133],{"class":207},[190,1299,1207],{"class":575},[190,1301,709],{"class":207},[190,1303,1305,1307,1309,1312,1315,1318,1321,1323,1325,1328,1331,1334,1337,1339,1341],{"class":192,"line":1304},65,[190,1306,1136],{"class":196},[190,1308,1202],{"class":203},[190,1310,1311],{"class":207},"(name).",[190,1313,1314],{"class":203},"map",[190,1316,1317],{"class":207},"((",[190,1319,1320],{"class":575},"part",[190,1322,682],{"class":207},[190,1324,673],{"class":196},[190,1326,1327],{"class":207}," client.",[190,1329,1330],{"class":203},"escapeIdentifier",[190,1332,1333],{"class":207},"(part)).",[190,1335,1336],{"class":203},"join",[190,1338,256],{"class":207},[190,1340,1230],{"class":353},[190,1342,312],{"class":207},[190,1344,1346],{"class":192,"line":1345},66,[190,1347,1189],{"class":207},[190,1349,1351],{"class":192,"line":1350},67,[190,1352,504],{"emptyLinePlaceholder":503},[190,1354,1356,1359,1362,1365,1367,1369,1371,1373],{"class":192,"line":1355},68,[190,1357,1358],{"class":196},"async",[190,1360,1361],{"class":196}," function",[190,1363,1364],{"class":203}," destinationColumns",[190,1366,256],{"class":207},[190,1368,1295],{"class":575},[190,1370,133],{"class":207},[190,1372,1207],{"class":575},[190,1374,709],{"class":207},[190,1376,1378,1380,1382,1385,1387,1390,1392,1394,1396],{"class":192,"line":1377},69,[190,1379,715],{"class":196},[190,1381,1260],{"class":207},[190,1383,1384],{"class":259},"schema",[190,1386,133],{"class":207},[190,1388,1389],{"class":259},"table",[190,1391,984],{"class":207},[190,1393,350],{"class":196},[190,1395,1202],{"class":203},[190,1397,1398],{"class":207},"(name);\n",[190,1400,1402,1404,1406,1409,1411,1413,1416,1418,1421],{"class":192,"line":1401},70,[190,1403,715],{"class":196},[190,1405,527],{"class":207},[190,1407,1408],{"class":259},"rows",[190,1410,533],{"class":207},[190,1412,350],{"class":196},[190,1414,1415],{"class":196}," await",[190,1417,1327],{"class":207},[190,1419,1420],{"class":203},"query",[190,1422,1423],{"class":207},"(\n",[190,1425,1427],{"class":192,"line":1426},71,[190,1428,1429],{"class":353},"        `SELECT column_name\n",[190,1431,1433],{"class":192,"line":1432},72,[190,1434,1435],{"class":353},"           FROM information_schema.columns\n",[190,1437,1439],{"class":192,"line":1438},73,[190,1440,1441],{"class":353},"          WHERE table_schema = $1 AND table_name = $2\n",[190,1443,1445],{"class":192,"line":1444},74,[190,1446,1447],{"class":353},"            AND is_generated = 'NEVER'\n",[190,1449,1451,1454],{"class":192,"line":1450},75,[190,1452,1453],{"class":353},"          ORDER BY ordinal_position`",[190,1455,223],{"class":207},[190,1457,1459],{"class":192,"line":1458},76,[190,1460,1461],{"class":207},"        [schema, table],\n",[190,1463,1465],{"class":192,"line":1464},77,[190,1466,1467],{"class":207},"    );\n",[190,1469,1471,1473,1476,1478,1480,1482],{"class":192,"line":1470},78,[190,1472,971],{"class":196},[190,1474,1475],{"class":207}," (rows.",[190,1477,780],{"class":259},[190,1479,1245],{"class":196},[190,1481,768],{"class":259},[190,1483,709],{"class":207},[190,1485,1487,1489,1491,1493,1495,1498,1500,1503],{"class":192,"line":1486},79,[190,1488,1004],{"class":196},[190,1490,838],{"class":196},[190,1492,841],{"class":203},[190,1494,256],{"class":207},[190,1496,1497],{"class":353},"`Destination table ${",[190,1499,1207],{"class":207},[190,1501,1502],{"class":353},"} not found or not visible to this role`",[190,1504,312],{"class":207},[190,1506,1508],{"class":192,"line":1507},80,[190,1509,965],{"class":207},[190,1511,1513,1515,1518,1520,1522,1525,1527,1529],{"class":192,"line":1512},81,[190,1514,1136],{"class":196},[190,1516,1517],{"class":207}," rows.",[190,1519,1314],{"class":203},[190,1521,1317],{"class":207},[190,1523,1524],{"class":575},"row",[190,1526,682],{"class":207},[190,1528,673],{"class":196},[190,1530,1531],{"class":207}," row.column_name);\n",[190,1533,1535],{"class":192,"line":1534},82,[190,1536,1189],{"class":207},[190,1538,1540],{"class":192,"line":1539},83,[190,1541,504],{"emptyLinePlaceholder":503},[190,1543,1545],{"class":192,"line":1544},84,[190,1546,1547],{"class":429},"\u002F\u002F One row in, one row out. Values are Postgres text (or null). Extend this for\n",[190,1549,1551],{"class":192,"line":1550},85,[190,1552,1553],{"class":429},"\u002F\u002F renames, filtering, or derived columns; never hold a reference to past rows.\n",[190,1555,1557,1559,1562,1564,1566,1568,1571],{"class":192,"line":1556},86,[190,1558,698],{"class":196},[190,1560,1561],{"class":203}," transformRow",[190,1563,256],{"class":207},[190,1565,1524],{"class":575},[190,1567,133],{"class":207},[190,1569,1570],{"class":575},"loadedAt",[190,1572,709],{"class":207},[190,1574,1576,1578,1580,1583],{"class":192,"line":1575},87,[190,1577,1136],{"class":196},[190,1579,527],{"class":207},[190,1581,1582],{"class":196},"...",[190,1584,1585],{"class":207},"row, loaded_at: loadedAt };\n",[190,1587,1589],{"class":192,"line":1588},88,[190,1590,1189],{"class":207},[190,1592,1594],{"class":192,"line":1593},89,[190,1595,504],{"emptyLinePlaceholder":503},[190,1597,1599],{"class":192,"line":1598},90,[190,1600,1601],{"class":429},"\u002F\u002F COPY text format: tab-delimited, \\N for null, and backslash, newline,\n",[190,1603,1605],{"class":192,"line":1604},91,[190,1606,1607],{"class":429},"\u002F\u002F carriage return, and the delimiter escaped with a backslash.\n",[190,1609,1611,1613,1616,1618,1620],{"class":192,"line":1610},92,[190,1612,698],{"class":196},[190,1614,1615],{"class":203}," toCopyField",[190,1617,256],{"class":207},[190,1619,679],{"class":575},[190,1621,709],{"class":207},[190,1623,1625,1627,1630,1632,1635,1637,1640,1642,1645],{"class":192,"line":1624},93,[190,1626,971],{"class":196},[190,1628,1629],{"class":207}," (value ",[190,1631,893],{"class":196},[190,1633,1634],{"class":259}," null",[190,1636,1089],{"class":196},[190,1638,1639],{"class":207}," value ",[190,1641,893],{"class":196},[190,1643,1644],{"class":259}," undefined",[190,1646,709],{"class":207},[190,1648,1650,1653,1656,1659,1662],{"class":192,"line":1649},94,[190,1651,1652],{"class":196},"        return",[190,1654,1655],{"class":353}," \"",[190,1657,1658],{"class":259},"\\\\",[190,1660,1661],{"class":353},"N\"",[190,1663,513],{"class":207},[190,1665,1667],{"class":192,"line":1666},95,[190,1668,965],{"class":207},[190,1670,1672,1674,1677],{"class":192,"line":1671},96,[190,1673,1136],{"class":196},[190,1675,1676],{"class":203}," String",[190,1678,1679],{"class":207},"(value)\n",[190,1681,1683,1686,1689,1691,1693,1696,1698,1701,1703,1706,1709,1711],{"class":192,"line":1682},97,[190,1684,1685],{"class":207},"        .",[190,1687,1688],{"class":203},"replace",[190,1690,256],{"class":207},[190,1692,77],{"class":353},[190,1694,1658],{"class":1695},"sRjNt",[190,1697,77],{"class":353},[190,1699,1700],{"class":196},"g",[190,1702,133],{"class":207},[190,1704,1705],{"class":353},"\"",[190,1707,1708],{"class":259},"\\\\\\\\",[190,1710,1705],{"class":353},[190,1712,1713],{"class":207},")\n",[190,1715,1717,1719,1721,1723,1725,1728,1730,1732,1734,1736,1738,1741],{"class":192,"line":1716},98,[190,1718,1685],{"class":207},[190,1720,1688],{"class":203},[190,1722,256],{"class":207},[190,1724,77],{"class":353},[190,1726,1727],{"class":259},"\\t",[190,1729,77],{"class":353},[190,1731,1700],{"class":196},[190,1733,133],{"class":207},[190,1735,1705],{"class":353},[190,1737,1658],{"class":259},[190,1739,1740],{"class":353},"t\"",[190,1742,1713],{"class":207},[190,1744,1746,1748,1750,1752,1754,1757,1759,1761,1763,1765,1767,1770],{"class":192,"line":1745},99,[190,1747,1685],{"class":207},[190,1749,1688],{"class":203},[190,1751,256],{"class":207},[190,1753,77],{"class":353},[190,1755,1756],{"class":259},"\\n",[190,1758,77],{"class":353},[190,1760,1700],{"class":196},[190,1762,133],{"class":207},[190,1764,1705],{"class":353},[190,1766,1658],{"class":259},[190,1768,1769],{"class":353},"n\"",[190,1771,1713],{"class":207},[190,1773,1775,1777,1779,1781,1783,1786,1788,1790,1792,1794,1796,1799],{"class":192,"line":1774},100,[190,1776,1685],{"class":207},[190,1778,1688],{"class":203},[190,1780,256],{"class":207},[190,1782,77],{"class":353},[190,1784,1785],{"class":259},"\\r",[190,1787,77],{"class":353},[190,1789,1700],{"class":196},[190,1791,133],{"class":207},[190,1793,1705],{"class":353},[190,1795,1658],{"class":259},[190,1797,1798],{"class":353},"r\"",[190,1800,312],{"class":207},[190,1802,1804],{"class":192,"line":1803},101,[190,1805,1189],{"class":207},[190,1807,1809],{"class":192,"line":1808},102,[190,1810,504],{"emptyLinePlaceholder":503},[190,1812,1814,1816,1818,1821],{"class":192,"line":1813},103,[190,1815,1358],{"class":196},[190,1817,1361],{"class":196},[190,1819,1820],{"class":203}," run",[190,1822,1823],{"class":207},"() {\n",[190,1825,1827,1829,1831,1834,1836,1839,1841,1844,1846,1849,1851,1854,1856,1858,1860,1863,1865,1867,1869],{"class":192,"line":1826},104,[190,1828,715],{"class":196},[190,1830,527],{"class":207},[190,1832,1833],{"class":259},"sourceTable",[190,1835,133],{"class":207},[190,1837,1838],{"class":259},"destTable",[190,1840,133],{"class":207},[190,1842,1843],{"class":259},"truncate",[190,1845,133],{"class":207},[190,1847,1848],{"class":259},"batchSize",[190,1850,133],{"class":207},[190,1852,1853],{"class":259},"logEvery",[190,1855,533],{"class":207},[190,1857,350],{"class":196},[190,1859,701],{"class":203},[190,1861,1862],{"class":207},"(process.argv.",[190,1864,876],{"class":203},[190,1866,256],{"class":207},[190,1868,266],{"class":259},[190,1870,1871],{"class":207},"));\n",[190,1873,1875],{"class":192,"line":1874},105,[190,1876,504],{"emptyLinePlaceholder":503},[190,1878,1880,1882,1885,1887,1889,1892,1895,1898,1901,1904],{"class":192,"line":1879},106,[190,1881,715],{"class":196},[190,1883,1884],{"class":259}," sourcePool",[190,1886,556],{"class":196},[190,1888,838],{"class":196},[190,1890,1891],{"class":203}," Pool",[190,1893,1894],{"class":207},"({ connectionString: process.env.",[190,1896,1897],{"class":259},"SOURCE_DATABASE_URL",[190,1899,1900],{"class":207},", max: ",[190,1902,1903],{"class":259},"1",[190,1905,1906],{"class":207}," });\n",[190,1908,1910,1912,1915,1917,1919,1921,1923,1926,1928,1930],{"class":192,"line":1909},107,[190,1911,715],{"class":196},[190,1913,1914],{"class":259}," destPool",[190,1916,556],{"class":196},[190,1918,838],{"class":196},[190,1920,1891],{"class":203},[190,1922,1894],{"class":207},[190,1924,1925],{"class":259},"DEST_DATABASE_URL",[190,1927,1900],{"class":207},[190,1929,1903],{"class":259},[190,1931,1906],{"class":207},[190,1933,1935,1937,1940,1942,1944,1947,1950],{"class":192,"line":1934},108,[190,1936,715],{"class":196},[190,1938,1939],{"class":259}," sourceClient",[190,1941,556],{"class":196},[190,1943,1415],{"class":196},[190,1945,1946],{"class":207}," sourcePool.",[190,1948,1949],{"class":203},"connect",[190,1951,1952],{"class":207},"();\n",[190,1954,1956,1958,1961,1963,1965,1968,1970],{"class":192,"line":1955},109,[190,1957,715],{"class":196},[190,1959,1960],{"class":259}," destClient",[190,1962,556],{"class":196},[190,1964,1415],{"class":196},[190,1966,1967],{"class":207}," destPool.",[190,1969,1949],{"class":203},[190,1971,1952],{"class":207},[190,1973,1975],{"class":192,"line":1974},110,[190,1976,504],{"emptyLinePlaceholder":503},[190,1978,1980,1982,1985,1987,1990,1993],{"class":192,"line":1979},111,[190,1981,715],{"class":196},[190,1983,1984],{"class":259}," startedAt",[190,1986,556],{"class":196},[190,1988,1989],{"class":207}," Date.",[190,1991,1992],{"class":203},"now",[190,1994,1952],{"class":207},[190,1996,1998,2000,2003,2005,2007,2010,2013,2016],{"class":192,"line":1997},112,[190,1999,715],{"class":196},[190,2001,2002],{"class":259}," loadedAt",[190,2004,556],{"class":196},[190,2006,838],{"class":196},[190,2008,2009],{"class":203}," Date",[190,2011,2012],{"class":207},"().",[190,2014,2015],{"class":203},"toISOString",[190,2017,1952],{"class":207},[190,2019,2021,2024,2027,2029,2031],{"class":192,"line":2020},113,[190,2022,2023],{"class":196},"    let",[190,2025,2026],{"class":207}," rowCount ",[190,2028,350],{"class":196},[190,2030,768],{"class":259},[190,2032,513],{"class":207},[190,2034,2036],{"class":192,"line":2035},114,[190,2037,504],{"emptyLinePlaceholder":503},[190,2039,2041,2044],{"class":192,"line":2040},115,[190,2042,2043],{"class":196},"    try",[190,2045,923],{"class":207},[190,2047,2049,2051,2054,2056,2058,2060],{"class":192,"line":2048},116,[190,2050,796],{"class":196},[190,2052,2053],{"class":259}," columns",[190,2055,556],{"class":196},[190,2057,1415],{"class":196},[190,2059,1364],{"class":203},[190,2061,2062],{"class":207},"(destClient, destTable);\n",[190,2064,2066,2068,2071,2073,2076,2078,2080,2083,2085,2087,2090,2092,2095,2097,2099,2102],{"class":192,"line":2065},117,[190,2067,796],{"class":196},[190,2069,2070],{"class":259}," columnList",[190,2072,556],{"class":196},[190,2074,2075],{"class":207}," columns.",[190,2077,1314],{"class":203},[190,2079,1317],{"class":207},[190,2081,2082],{"class":575},"c",[190,2084,682],{"class":207},[190,2086,673],{"class":196},[190,2088,2089],{"class":207}," destClient.",[190,2091,1330],{"class":203},[190,2093,2094],{"class":207},"(c)).",[190,2096,1336],{"class":203},[190,2098,256],{"class":207},[190,2100,2101],{"class":353},"\", \"",[190,2103,312],{"class":207},[190,2105,2107,2110,2113,2115,2118,2120,2123,2125,2128,2131,2133,2135,2138],{"class":192,"line":2106},118,[190,2108,2109],{"class":207},"        console.",[190,2111,2112],{"class":203},"log",[190,2114,256],{"class":207},[190,2116,2117],{"class":353},"`Streaming ${",[190,2119,1833],{"class":207},[190,2121,2122],{"class":353},"} -> ${",[190,2124,1838],{"class":207},[190,2126,2127],{"class":353},"} (${",[190,2129,2130],{"class":207},"columns",[190,2132,149],{"class":353},[190,2134,780],{"class":259},[190,2136,2137],{"class":353},"} columns)`",[190,2139,312],{"class":207},[190,2141,2143],{"class":192,"line":2142},119,[190,2144,504],{"emptyLinePlaceholder":503},[190,2146,2148,2151,2153,2155,2157,2160],{"class":192,"line":2147},120,[190,2149,2150],{"class":196},"        await",[190,2152,2089],{"class":207},[190,2154,1420],{"class":203},[190,2156,256],{"class":207},[190,2158,2159],{"class":353},"\"BEGIN\"",[190,2161,312],{"class":207},[190,2163,2165,2167],{"class":192,"line":2164},121,[190,2166,810],{"class":196},[190,2168,2169],{"class":207}," (truncate) {\n",[190,2171,2173,2176,2178,2180,2182,2185,2188,2190,2193,2195,2197,2200,2202],{"class":192,"line":2172},122,[190,2174,2175],{"class":196},"            await",[190,2177,2089],{"class":207},[190,2179,1420],{"class":203},[190,2181,256],{"class":207},[190,2183,2184],{"class":353},"`TRUNCATE ${",[190,2186,2187],{"class":203},"quoteName",[190,2189,256],{"class":353},[190,2191,2192],{"class":207},"destClient",[190,2194,133],{"class":353},[190,2196,1838],{"class":207},[190,2198,2199],{"class":353},")",[190,2201,852],{"class":353},[190,2203,312],{"class":207},[190,2205,2207],{"class":192,"line":2206},123,[190,2208,860],{"class":207},[190,2210,2212],{"class":192,"line":2211},124,[190,2213,504],{"emptyLinePlaceholder":503},[190,2215,2217,2219,2222,2224,2227,2229],{"class":192,"line":2216},125,[190,2218,796],{"class":196},[190,2220,2221],{"class":259}," source",[190,2223,556],{"class":196},[190,2225,2226],{"class":207}," sourceClient.",[190,2228,1420],{"class":203},[190,2230,1423],{"class":207},[190,2232,2234,2237,2239,2241,2244,2246,2248,2251,2253,2255,2257,2259],{"class":192,"line":2233},126,[190,2235,2236],{"class":196},"            new",[190,2238,553],{"class":203},[190,2240,256],{"class":207},[190,2242,2243],{"class":353},"`SELECT * FROM ${",[190,2245,2187],{"class":203},[190,2247,256],{"class":353},[190,2249,2250],{"class":207},"sourceClient",[190,2252,133],{"class":353},[190,2254,1833],{"class":207},[190,2256,2199],{"class":353},[190,2258,852],{"class":353},[190,2260,2261],{"class":207},", [], {\n",[190,2263,2265],{"class":192,"line":2264},127,[190,2266,2267],{"class":207},"                batchSize,\n",[190,2269,2271,2274,2277],{"class":192,"line":2270},128,[190,2272,2273],{"class":207},"                types: ",[190,2275,2276],{"class":259},"RAW_TEXT",[190,2278,223],{"class":207},[190,2280,2282],{"class":192,"line":2281},129,[190,2283,2284],{"class":207},"            }),\n",[190,2286,2288],{"class":192,"line":2287},130,[190,2289,2290],{"class":207},"        );\n",[190,2292,2294],{"class":192,"line":2293},131,[190,2295,504],{"emptyLinePlaceholder":503},[190,2297,2299,2301,2304,2306,2308,2311],{"class":192,"line":2298},132,[190,2300,796],{"class":196},[190,2302,2303],{"class":259}," toCopyLines",[190,2305,556],{"class":196},[190,2307,838],{"class":196},[190,2309,2310],{"class":203}," Transform",[190,2312,2313],{"class":207},"({\n",[190,2315,2317,2320,2323],{"class":192,"line":2316},133,[190,2318,2319],{"class":207},"            writableObjectMode: ",[190,2321,2322],{"class":259},"true",[190,2324,223],{"class":207},[190,2326,2328,2331,2333],{"class":192,"line":2327},134,[190,2329,2330],{"class":207},"            readableObjectMode: ",[190,2332,726],{"class":259},[190,2334,223],{"class":207},[190,2336,2338,2341,2343,2345,2347,2350,2352,2355],{"class":192,"line":2337},135,[190,2339,2340],{"class":203},"            transform",[190,2342,256],{"class":207},[190,2344,1524],{"class":575},[190,2346,133],{"class":207},[190,2348,2349],{"class":575},"_encoding",[190,2351,133],{"class":207},[190,2353,2354],{"class":575},"callback",[190,2356,709],{"class":207},[190,2358,2360,2363,2366,2368,2370],{"class":192,"line":2359},136,[190,2361,2362],{"class":196},"                const",[190,2364,2365],{"class":259}," out",[190,2367,556],{"class":196},[190,2369,1561],{"class":203},[190,2371,2372],{"class":207},"(row, loadedAt);\n",[190,2374,2376,2379,2382,2384,2386],{"class":192,"line":2375},137,[190,2377,2378],{"class":196},"                if",[190,2380,2381],{"class":207}," (rowCount ",[190,2383,893],{"class":196},[190,2385,768],{"class":259},[190,2387,709],{"class":207},[190,2389,2391,2394,2397,2399,2401,2404,2406,2408,2410,2412,2414,2417,2420],{"class":192,"line":2390},138,[190,2392,2393],{"class":196},"                    const",[190,2395,2396],{"class":259}," missing",[190,2398,556],{"class":196},[190,2400,2075],{"class":207},[190,2402,2403],{"class":203},"filter",[190,2405,1317],{"class":207},[190,2407,2082],{"class":575},[190,2409,682],{"class":207},[190,2411,673],{"class":196},[190,2413,990],{"class":196},[190,2415,2416],{"class":207},"(c ",[190,2418,2419],{"class":196},"in",[190,2421,2422],{"class":207}," out));\n",[190,2424,2426,2429,2432,2434,2437,2439],{"class":192,"line":2425},139,[190,2427,2428],{"class":196},"                    if",[190,2430,2431],{"class":207}," (missing.",[190,2433,780],{"class":259},[190,2435,2436],{"class":196}," >",[190,2438,768],{"class":259},[190,2440,709],{"class":207},[190,2442,2444,2447,2449,2452,2454,2456,2459,2462,2464,2466,2468,2470,2472,2474],{"class":192,"line":2443},140,[190,2445,2446],{"class":203},"                        callback",[190,2448,256],{"class":207},[190,2450,2451],{"class":196},"new",[190,2453,841],{"class":203},[190,2455,256],{"class":207},[190,2457,2458],{"class":353},"`Transform does not produce destination column(s): ${",[190,2460,2461],{"class":207},"missing",[190,2463,149],{"class":353},[190,2465,1336],{"class":203},[190,2467,256],{"class":353},[190,2469,2101],{"class":353},[190,2471,2199],{"class":353},[190,2473,852],{"class":353},[190,2475,1871],{"class":207},[190,2477,2479,2482],{"class":192,"line":2478},141,[190,2480,2481],{"class":196},"                        return",[190,2483,513],{"class":207},[190,2485,2487],{"class":192,"line":2486},142,[190,2488,2489],{"class":207},"                    }\n",[190,2491,2493],{"class":192,"line":2492},143,[190,2494,2495],{"class":207},"                }\n",[190,2497,2499,2502,2504,2506],{"class":192,"line":2498},144,[190,2500,2501],{"class":207},"                rowCount ",[190,2503,785],{"class":196},[190,2505,788],{"class":259},[190,2507,513],{"class":207},[190,2509,2511,2513,2515,2518,2520,2522,2524],{"class":192,"line":2510},145,[190,2512,2378],{"class":196},[190,2514,2381],{"class":207},[190,2516,2517],{"class":196},"%",[190,2519,1103],{"class":207},[190,2521,893],{"class":196},[190,2523,768],{"class":259},[190,2525,709],{"class":207},[190,2527,2529,2531,2534,2536,2539,2542,2545,2548,2551,2553,2556,2559,2561],{"class":192,"line":2528},146,[190,2530,2393],{"class":196},[190,2532,2533],{"class":259}," heapMb",[190,2535,556],{"class":196},[190,2537,2538],{"class":207}," Math.",[190,2540,2541],{"class":203},"round",[190,2543,2544],{"class":207},"(process.",[190,2546,2547],{"class":203},"memoryUsage",[190,2549,2550],{"class":207},"().heapUsed ",[190,2552,77],{"class":196},[190,2554,2555],{"class":259}," 1024",[190,2557,2558],{"class":196}," \u002F",[190,2560,2555],{"class":259},[190,2562,312],{"class":207},[190,2564,2566,2569,2571,2573,2576,2579,2581,2584,2587,2590,2593,2596],{"class":192,"line":2565},147,[190,2567,2568],{"class":207},"                    console.",[190,2570,2112],{"class":203},[190,2572,256],{"class":207},[190,2574,2575],{"class":353},"`Processed ${",[190,2577,2578],{"class":207},"rowCount",[190,2580,149],{"class":353},[190,2582,2583],{"class":203},"toLocaleString",[190,2585,2586],{"class":353},"()",[190,2588,2589],{"class":353},"} rows (heap ${",[190,2591,2592],{"class":207},"heapMb",[190,2594,2595],{"class":353},"} MB)`",[190,2597,312],{"class":207},[190,2599,2601],{"class":192,"line":2600},148,[190,2602,2495],{"class":207},[190,2604,2606,2609,2611,2614,2617,2619,2621,2623,2625,2627,2629,2632,2634,2636,2638,2640,2642,2644,2646,2648,2650,2652],{"class":192,"line":2605},149,[190,2607,2608],{"class":203},"                callback",[190,2610,256],{"class":207},[190,2612,2613],{"class":259},"null",[190,2615,2616],{"class":207},", columns.",[190,2618,1314],{"class":203},[190,2620,1317],{"class":207},[190,2622,2082],{"class":575},[190,2624,682],{"class":207},[190,2626,673],{"class":196},[190,2628,1615],{"class":203},[190,2630,2631],{"class":207},"(out[c])).",[190,2633,1336],{"class":203},[190,2635,256],{"class":207},[190,2637,1705],{"class":353},[190,2639,1727],{"class":259},[190,2641,1705],{"class":353},[190,2643,682],{"class":207},[190,2645,937],{"class":196},[190,2647,1655],{"class":353},[190,2649,1756],{"class":259},[190,2651,1705],{"class":353},[190,2653,312],{"class":207},[190,2655,2657],{"class":192,"line":2656},150,[190,2658,2659],{"class":207},"            },\n",[190,2661,2663],{"class":192,"line":2662},151,[190,2664,2665],{"class":207},"        });\n",[190,2667,2669],{"class":192,"line":2668},152,[190,2670,504],{"emptyLinePlaceholder":503},[190,2672,2674,2676,2679,2681,2683,2685],{"class":192,"line":2673},153,[190,2675,796],{"class":196},[190,2677,2678],{"class":259}," sink",[190,2680,556],{"class":196},[190,2682,2089],{"class":207},[190,2684,1420],{"class":203},[190,2686,1423],{"class":207},[190,2688,2690,2693,2695,2698,2700,2702,2704,2706,2708,2710,2712,2715,2718],{"class":192,"line":2689},154,[190,2691,2692],{"class":203},"            copyFrom",[190,2694,256],{"class":207},[190,2696,2697],{"class":353},"`COPY ${",[190,2699,2187],{"class":203},[190,2701,256],{"class":353},[190,2703,2192],{"class":207},[190,2705,133],{"class":353},[190,2707,1838],{"class":207},[190,2709,2199],{"class":353},[190,2711,2127],{"class":353},[190,2713,2714],{"class":207},"columnList",[190,2716,2717],{"class":353},"}) FROM STDIN`",[190,2719,269],{"class":207},[190,2721,2723],{"class":192,"line":2722},155,[190,2724,2290],{"class":207},[190,2726,2728],{"class":192,"line":2727},156,[190,2729,504],{"emptyLinePlaceholder":503},[190,2731,2733],{"class":192,"line":2732},157,[190,2734,2735],{"class":429},"        \u002F\u002F Propagates backpressure and destroys every stream if any one fails;\n",[190,2737,2739],{"class":192,"line":2738},158,[190,2740,2741],{"class":429},"        \u002F\u002F destroying the COPY stream sends CopyFail so the server aborts the COPY.\n",[190,2743,2745,2747,2750],{"class":192,"line":2744},159,[190,2746,2150],{"class":196},[190,2748,2749],{"class":203}," pipeline",[190,2751,2752],{"class":207},"(source, toCopyLines, sink);\n",[190,2754,2756,2758,2760,2762,2764,2767],{"class":192,"line":2755},160,[190,2757,2150],{"class":196},[190,2759,2089],{"class":207},[190,2761,1420],{"class":203},[190,2763,256],{"class":207},[190,2765,2766],{"class":353},"\"COMMIT\"",[190,2768,312],{"class":207},[190,2770,2772],{"class":192,"line":2771},161,[190,2773,504],{"emptyLinePlaceholder":503},[190,2775,2777,2779,2782,2784,2787,2789,2792,2795,2798,2800,2803,2806,2809,2811,2813],{"class":192,"line":2776},162,[190,2778,796],{"class":196},[190,2780,2781],{"class":259}," seconds",[190,2783,556],{"class":196},[190,2785,2786],{"class":207}," ((Date.",[190,2788,1992],{"class":203},[190,2790,2791],{"class":207},"() ",[190,2793,2794],{"class":196},"-",[190,2796,2797],{"class":207}," startedAt) ",[190,2799,77],{"class":196},[190,2801,2802],{"class":259}," 1000",[190,2804,2805],{"class":207},").",[190,2807,2808],{"class":203},"toFixed",[190,2810,256],{"class":207},[190,2812,1903],{"class":259},[190,2814,312],{"class":207},[190,2816,2818,2820,2822,2824,2827,2829,2831,2833,2835,2838,2841,2844],{"class":192,"line":2817},163,[190,2819,2109],{"class":207},[190,2821,2112],{"class":203},[190,2823,256],{"class":207},[190,2825,2826],{"class":353},"`Done: ${",[190,2828,2578],{"class":207},[190,2830,149],{"class":353},[190,2832,2583],{"class":203},[190,2834,2586],{"class":353},[190,2836,2837],{"class":353},"} rows in ${",[190,2839,2840],{"class":207},"seconds",[190,2842,2843],{"class":353},"}s, committed`",[190,2845,312],{"class":207},[190,2847,2849,2852,2855],{"class":192,"line":2848},164,[190,2850,2851],{"class":207},"    } ",[190,2853,2854],{"class":196},"catch",[190,2856,2857],{"class":207}," (error) {\n",[190,2859,2861,2863,2865,2867,2869,2872,2874,2876,2879,2881],{"class":192,"line":2860},165,[190,2862,2150],{"class":196},[190,2864,2089],{"class":207},[190,2866,1420],{"class":203},[190,2868,256],{"class":207},[190,2870,2871],{"class":353},"\"ROLLBACK\"",[190,2873,2805],{"class":207},[190,2875,2854],{"class":203},[190,2877,2878],{"class":207},"(() ",[190,2880,673],{"class":196},[190,2882,2883],{"class":207}," {});\n",[190,2885,2887,2889],{"class":192,"line":2886},166,[190,2888,1004],{"class":196},[190,2890,2891],{"class":207}," error;\n",[190,2893,2895,2897,2900],{"class":192,"line":2894},167,[190,2896,2851],{"class":207},[190,2898,2899],{"class":196},"finally",[190,2901,923],{"class":207},[190,2903,2905,2908,2911],{"class":192,"line":2904},168,[190,2906,2907],{"class":207},"        sourceClient.",[190,2909,2910],{"class":203},"release",[190,2912,1952],{"class":207},[190,2914,2916,2919,2921],{"class":192,"line":2915},169,[190,2917,2918],{"class":207},"        destClient.",[190,2920,2910],{"class":203},[190,2922,1952],{"class":207},[190,2924,2926,2928,2931,2933,2936,2939,2942,2945,2947],{"class":192,"line":2925},170,[190,2927,2150],{"class":196},[190,2929,2930],{"class":259}," Promise",[190,2932,149],{"class":207},[190,2934,2935],{"class":203},"all",[190,2937,2938],{"class":207},"([sourcePool.",[190,2940,2941],{"class":203},"end",[190,2943,2944],{"class":207},"(), destPool.",[190,2946,2941],{"class":203},[190,2948,2949],{"class":207},"()]);\n",[190,2951,2953],{"class":192,"line":2952},171,[190,2954,965],{"class":207},[190,2956,2958],{"class":192,"line":2957},172,[190,2959,1189],{"class":207},[190,2961,2963],{"class":192,"line":2962},173,[190,2964,504],{"emptyLinePlaceholder":503},[190,2966,2968,2971,2973,2975,2977,2980,2982,2984],{"class":192,"line":2967},174,[190,2969,2970],{"class":203},"run",[190,2972,2012],{"class":207},[190,2974,2854],{"class":203},[190,2976,1317],{"class":207},[190,2978,2979],{"class":575},"error",[190,2981,682],{"class":207},[190,2983,673],{"class":196},[190,2985,923],{"class":207},[190,2987,2989,2992,2994,2996,2999],{"class":192,"line":2988},175,[190,2990,2991],{"class":207},"    console.",[190,2993,2979],{"class":203},[190,2995,256],{"class":207},[190,2997,2998],{"class":353},"\"ETL job failed:\"",[190,3000,3001],{"class":207},", error.message);\n",[190,3003,3005,3008,3010,3012],{"class":192,"line":3004},176,[190,3006,3007],{"class":207},"    process.exitCode ",[190,3009,350],{"class":196},[190,3011,788],{"class":259},[190,3013,513],{"class":207},[190,3015,3017],{"class":192,"line":3016},177,[190,3018,3019],{"class":207},"});\n",[120,3021,3023],{"id":3022},"notes","Notes",[31,3025,3026,3063,3108,3135,3153,3169,3189],{},[34,3027,3028,40,3031,3033,3034,3037,3038,3040,3041,3044,3045,3047,3048,3051,3052,133,3054,96,3056,3059,3060,3062],{},[37,3029,3030],{},"Where the memory bound comes from.",[15,3032,136],{}," wraps ",[15,3035,3036],{},"pg-cursor",", which fetches ",[15,3039,1848],{}," rows per round trip instead of the whole result set. The readable side's ",[15,3042,3043],{},"highWaterMark"," defaults to the same ",[15,3046,1848],{},", so the stream buffers about one batch before it stops pulling. The package README describes it as keeping \"only a low number of rows in memory\", which is the property you want. The bound is on rows, not bytes: the working set is roughly ",[15,3049,3050],{},"--batch-size"," times the average row width, plus the transform's and COPY stream's own small buffers, whatever the table size. A table with wide rows (large ",[15,3053,404],{},[15,3055,80],{},[15,3057,3058],{},"bytea"," values) holds far more memory per batch than one of narrow rows, so lower ",[15,3061,3050],{}," for those.",[34,3064,3065,3074,3075,3078,3079,3082,3083,3085,3086,3088,3089,3091,3092,3095,3096,3098,3099,77,3102,3105,3106,149],{},[37,3066,3067,3068,3070,3071,149],{},"Why ",[15,3069,43],{}," and not ",[15,3072,3073],{},".pipe()"," Node's docs say ",[15,3076,3077],{},"stream.pipeline()"," calls ",[15,3080,3081],{},"stream.destroy(err)"," on every stream in the chain when one fails. Hand-wired ",[15,3084,3073],{}," calls don't do that, which is how a failed transform leaves a source cursor open or a ",[15,3087,51],{}," hanging. With pg-copy-streams 6 and later, destroying the ",[15,3090,51],{}," stream sends ",[15,3093,3094],{},"CopyFail",", which aborts the ",[15,3097,51],{},". The explicit ",[15,3100,3101],{},"BEGIN",[15,3103,3104],{},"ROLLBACK"," around it also undoes the ",[15,3107,163],{},[34,3109,3110,3113,3114,3117,3118,3121,3122,3124,3125,3127,3128,3130,3131,3134],{},[37,3111,3112],{},"Detecting the end of the COPY."," Since pg-copy-streams 4.0, a ",[15,3115,3116],{},"COPY FROM"," stream signals completion with the standard ",[15,3119,3120],{},"finish"," event, not ",[15,3123,2941],{},". ",[15,3126,43],{}," already waits for ",[15,3129,3120],{},", so the ",[15,3132,3133],{},"COMMIT"," only runs after the server has accepted every row.",[34,3136,3137,3144,3145,3148,3149,3152],{},[37,3138,3139,3140,3143],{},"Don't use ",[15,3141,3142],{},"pool.query()"," for COPY."," The pg-copy-streams README is explicit: the pool's convenience method returns a Promise and doesn't support custom query objects. Take a client with ",[15,3146,3147],{},"pool.connect()"," and call ",[15,3150,3151],{},"client.query()"," on it, as the script does.",[34,3154,3155,3158,3159,3161,3162,3165,3166,3168],{},[37,3156,3157],{},"Failed COPYs leave dead space."," PostgreSQL stops a ",[15,3160,3116],{}," at the first bad row. Rows already sent aren't visible, but they still take disk space until ",[15,3163,3164],{},"VACUUM"," reclaims it. A large load that fails near the end is worth a manual ",[15,3167,3164],{}," on the destination table.",[34,3170,3171,3174,3175,3178,3179,137,3182,77,3185,3188],{},[37,3172,3173],{},"Raw text has one catch."," Because values are Postgres text, the transform works with strings: ",[15,3176,3177],{},"\"12.50\"",", not ",[15,3180,3181],{},"12.5",[15,3183,3184],{},"\"t\"",[15,3186,3187],{},"\"f\""," for booleans. Parse only the columns you actually compute on, and turn them back into strings Postgres accepts before returning. That's also why no conversion is needed on the way out: the COPY docs define text-format values as the strings produced by each type's output function, or accepted by its input function.",[34,3190,3191,3194,3195,3197,3198,3203],{},[37,3192,3193],{},"Consistent snapshot."," A single ",[15,3196,155],{}," sees one snapshot of the source for its whole run, even while the table keeps changing. The cost is that a long read on a busy primary holds back cleanup of dead rows until it finishes. Run big extracts against a replica, or use the ",[3199,3200,3202],"a",{"href":3201},"\u002F2025\u002F12\u002F17\u002Fpython-data-warehouse-incremental-loads-from-postgres-to-bigquery\u002F","watermark approach"," so each run only reads recent changes.",[120,3205,3207],{"id":3206},"references","References",[31,3209,3210,3218,3225,3232,3242,3249],{},[34,3211,3212],{},[3199,3213,3217],{"href":3214,"rel":3215},"https:\u002F\u002Fgithub.com\u002Fbrianc\u002Fnode-postgres\u002Ftree\u002Fmaster\u002Fpackages\u002Fpg-query-stream",[3216],"nofollow","pg-query-stream README",[34,3219,3220],{},[3199,3221,3224],{"href":3222,"rel":3223},"https:\u002F\u002Fgithub.com\u002Fbrianc\u002Fnode-pg-copy-streams",[3216],"pg-copy-streams README",[34,3226,3227],{},[3199,3228,3231],{"href":3229,"rel":3230},"https:\u002F\u002Fnode-postgres.com\u002Ffeatures\u002Ftypes",[3216],"node-postgres: data types and type parsing",[34,3233,3234],{},[3199,3235,3238,3239,3241],{"href":3236,"rel":3237},"https:\u002F\u002Fnodejs.org\u002Fapi\u002Fstream.html",[3216],"Node.js stream docs: ",[15,3240,3077],{}," and backpressure",[34,3243,3244],{},[3199,3245,3248],{"href":3246,"rel":3247},"https:\u002F\u002Fwww.postgresql.org\u002Fdocs\u002Fcurrent\u002Fsql-copy.html",[3216],"PostgreSQL: COPY (text format and escaping rules)",[34,3250,3251],{},[3199,3252,3255],{"href":3253,"rel":3254},"https:\u002F\u002Fwww.postgresql.org\u002Fdocs\u002Fcurrent\u002Finfoschema-columns.html",[3216],"PostgreSQL: information_schema.columns",[3257,3258,3259],"style",{},"html pre.shiki code .snl16, html code.shiki .snl16{--shiki-default:#F97583}html pre.shiki code .svObZ, html code.shiki .svObZ{--shiki-default:#B392F0}html pre.shiki code .s95oV, html code.shiki .s95oV{--shiki-default:#E1E4E8}html pre.shiki code .sDLfK, html code.shiki .sDLfK{--shiki-default:#79B8FF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html pre.shiki code .sU2Wk, html code.shiki .sU2Wk{--shiki-default:#9ECBFF}html pre.shiki code .sAwPA, html code.shiki .sAwPA{--shiki-default:#6A737D}html pre.shiki code .s9osk, html code.shiki .s9osk{--shiki-default:#FFAB70}html pre.shiki code .sRjNt, html code.shiki .sRjNt{--shiki-default:#85E89D;--shiki-default-font-weight:bold}",{"title":186,"searchDepth":211,"depth":211,"links":3261},[3262,3263,3264,3265,3266],{"id":122,"depth":211,"text":123},{"id":315,"depth":211,"text":316},{"id":416,"depth":211,"text":417},{"id":3022,"depth":211,"text":3023},{"id":3206,"depth":211,"text":3207},"techcolumnist",[3269,3270],"scripts","engineering","2026-02-25T14:00:00Z","A Node.js script that streams Postgres rows through a cursor, a transform and COPY FROM STDIN with backpressure, so memory tracks batch size, not table size.","md",false,null,{},"\u002Fblog\u002F2026\u002F02\u002F25\u002Fnode-js-data-warehouse-streaming-etl-rows-without-loading-them-all-in-memory",{"title":6,"description":3272},[3267],"blog\u002F2026\u002F02\u002F25\u002Fnode-js-data-warehouse-streaming-etl-rows-without-loading-them-all-in-memory",[3282,3283,3284,185],"nodejs","data-warehouse","etl","\u002F2026\u002F02\u002F25\u002Fnode-js-data-warehouse-streaming-etl-rows-without-loading-them-all-in-memory\u002F","TtrChrfhEJcPtOoO7MpghNR8GzSeDiplnbJzOjoRebQ",{"title":3288,"description":3289,"date":3290,"url":3291,"categories":3292,"tags":3293,"image":3275,"readingTime":236,"canonical":3267,"sites":3295,"series":3275,"seriesOrder":3275},"Node.js: gcloud – Rotating Service Account Keys Before They Expire","A Node.js script that finds aging Google Cloud service account keys, mints one replacement, and disables the old keys once it is deployed.","2026-03-04T14:00:00Z","\u002F2026\u002F03\u002F04\u002Fnode-js-gcloud-rotating-service-account-keys-before-they-expire\u002F",[3269,3270],[3282,3294],"gcloud",[3267],{"title":3297,"description":3298,"date":3299,"url":3300,"categories":3301,"tags":3302,"image":3275,"readingTime":247,"canonical":3267,"sites":3304,"series":3275,"seriesOrder":3275},"Node.js: Azure – A Slack Bot for Resource Group Budget Alerts","A Node.js script that queries month-to-date Azure cost per resource group, projects month-end spend, and posts Block Kit messages to Slack for groups over budget.","2026-02-18T14:00:00Z","\u002F2026\u002F02\u002F18\u002Fnode-js-azure-a-slack-bot-for-resource-group-budget-alerts\u002F",[3269,3270],[3282,3303],"azure",[3267],[3306,3315,3322],{"title":3307,"description":3308,"date":3309,"url":3310,"categories":3311,"tags":3312,"image":3275,"readingTime":226,"canonical":3267,"sites":3314,"series":3275,"seriesOrder":3275},"Python: Data Warehouse – Catching Schema Drift Before an ETL Run","A Python pre-flight check that snapshots Postgres column definitions from information_schema and fails the ETL run when a column is dropped, retyped, or resized.","2026-01-21T14:00:00Z","\u002F2026\u002F01\u002F21\u002Fpython-data-warehouse-catching-schema-drift-before-an-etl-run\u002F",[3269,3270],[3313,3283,3284,185],"python",[3267],{"title":3316,"description":3317,"date":3318,"url":3201,"categories":3319,"tags":3320,"image":3275,"readingTime":247,"canonical":3267,"sites":3321,"series":3275,"seriesOrder":3275},"Python: Data Warehouse – Incremental Loads from Postgres to BigQuery","A Python script that copies new and changed Postgres rows into BigQuery with one load job and a MERGE, advancing the watermark in the same transaction.","2025-12-17T14:00:00Z",[3269,3270],[3313,3283,3284,3294,185],[3267],{"title":3323,"description":3324,"date":3325,"url":3326,"categories":3327,"tags":3328,"image":3275,"readingTime":278,"canonical":3267,"sites":3330,"series":3275,"seriesOrder":3275},"Data Warehouse: Reusable SQL Models for IT Operations Metrics","Layered, version-controlled SQL models for MTTR, ticket aging, SLA compliance, and patch compliance, with plain-SQL tests and the dbt equivalent.","2026-08-19T14:00:00Z","\u002F2026\u002F08\u002F19\u002Fdata-warehouse-reusable-sql-models-for-it-operations-metrics\u002F",[3270],[3283,3329,185],"reporting",[3267],{"doc":3275,"posts":3332},[],1790052513649]