[{"data":1,"prerenderedAt":3729},["ShallowReactive",2],{"post:\u002F2025\u002F12\u002F17\u002Fpython-data-warehouse-incremental-loads-from-postgres-to-bigquery\u002F":3},{"post":4,"newer":3682,"older":3691,"related":3701,"series":3727},{"id":5,"title":6,"body":7,"canonical":3662,"categories":3663,"date":3666,"description":3667,"extension":3668,"featured":3669,"hero":3670,"image":3670,"meta":3671,"navigation":555,"path":3672,"readingTime":141,"seo":3673,"series":3670,"seriesOrder":3670,"sites":3674,"source":3670,"stem":3675,"tags":3676,"updated":3670,"url":3680,"__hash__":3681},"blog\u002Fblog\u002F2025\u002F12\u002F17\u002Fpython-data-warehouse-incremental-loads-from-postgres-to-bigquery.md","Python: Data Warehouse – Incremental Loads from Postgres to BigQuery",{"type":8,"value":9,"toc":3655},"minimark",[10,19,22,48,55,60,301,305,308,405,427,430,438,441,447,461,507,514,526,530,3444,3448,3571,3575,3651],[11,12,13,14,18],"p",{},"A full-refresh load from an operational Postgres database into BigQuery works fine right up until the source table gets big enough that re-copying the whole thing every run starts costing real time and real money. The usual fix is a watermark: track the highest value of an ",[15,16,17],"code",{},"updated_at"," (or similar) column you've already loaded, and on the next run only pull rows newer than that. It sounds simple, but the parts that actually matter are where the watermark lives and when it gets advanced. Advance it before the data commits and a failed run silently skips rows forever. Advance it separately from the data and a crash between the two steps leaves them disagreeing.",[11,20,21],{},"This script does the whole run as three steps:",[23,24,25,29,41],"ol",{},[26,27,28],"li",{},"Extract rows newer than the watermark (minus a small overlap window) from Postgres through a server-side cursor, writing them to a local newline-delimited JSON file so memory stays flat.",[26,30,31,32,36,37,40],{},"Load that file into a staging table with ",[33,34,35],"strong",{},"one"," BigQuery load job (",[15,38,39],{},"WRITE_TRUNCATE","), which is free on the default shared slot pool.",[26,42,43,44,47],{},"Run a single multi-statement transaction that ",[15,45,46],{},"MERGE","s staging into the destination and advances the watermark. Either both happen or neither does.",[11,49,50,51,54],{},"The obvious first draft of this job appends every 5,000-row batch with its own load job and records the watermark with ",[15,52,53],{},"insert_rows_json",". Both choices cause trouble, covered in the Notes below.",[56,57,59],"h2",{"id":58},"requirements","Requirements",[61,62,63,66,80,203,269,279,294],"ul",{},[26,64,65],{},"Python 3.10 or later.",[26,67,68,71,72,75,76,79],{},[15,69,70],{},"psycopg2-binary"," 2.9.1 or later (the first release with Python 3.10 wheels) and ",[15,73,74],{},"google-cloud-bigquery"," (",[15,77,78],{},"pip install psycopg2-binary google-cloud-bigquery",").",[26,81,82,83],{},"A destination table that already exists in BigQuery with the columns you want to copy. The script selects exactly those columns from Postgres, so extra source columns are ignored and a missing one fails the extract loudly instead of loading nulls. For example:",[84,85,90],"pre",{"className":86,"code":87,"language":88,"meta":89,"style":89},"language-sql shiki shiki-themes github-dark","CREATE TABLE `my-project.warehouse.orders` (\n  id INT64 NOT NULL,\n  customer_id INT64,\n  status STRING,\n  total NUMERIC,\n  created_at TIMESTAMP,\n  updated_at TIMESTAMP\n)\nPARTITION BY DATE(created_at)\nCLUSTER BY id;\n","sql","",[15,91,92,112,124,130,139,150,161,170,176,191],{"__ignoreMap":89},[93,94,97,101,104,108],"span",{"class":95,"line":96},"line",1,[93,98,100],{"class":99},"snl16","CREATE",[93,102,103],{"class":99}," TABLE",[93,105,107],{"class":106},"sU2Wk"," `my-project.warehouse.orders`",[93,109,111],{"class":110},"s95oV"," (\n",[93,113,115,118,121],{"class":95,"line":114},2,[93,116,117],{"class":110},"  id INT64 ",[93,119,120],{"class":99},"NOT NULL",[93,122,123],{"class":110},",\n",[93,125,127],{"class":95,"line":126},3,[93,128,129],{"class":110},"  customer_id INT64,\n",[93,131,133,136],{"class":95,"line":132},4,[93,134,135],{"class":99},"  status",[93,137,138],{"class":110}," STRING,\n",[93,140,142,145,148],{"class":95,"line":141},5,[93,143,144],{"class":110},"  total ",[93,146,147],{"class":99},"NUMERIC",[93,149,123],{"class":110},[93,151,153,156,159],{"class":95,"line":152},6,[93,154,155],{"class":110},"  created_at ",[93,157,158],{"class":99},"TIMESTAMP",[93,160,123],{"class":110},[93,162,164,167],{"class":95,"line":163},7,[93,165,166],{"class":110},"  updated_at ",[93,168,169],{"class":99},"TIMESTAMP\n",[93,171,173],{"class":95,"line":172},8,[93,174,175],{"class":110},")\n",[93,177,179,182,185,188],{"class":95,"line":178},9,[93,180,181],{"class":99},"PARTITION",[93,183,184],{"class":99}," BY",[93,186,187],{"class":99}," DATE",[93,189,190],{"class":110},"(created_at)\n",[93,192,194,197,200],{"class":95,"line":193},10,[93,195,196],{"class":110},"CLUSTER ",[93,198,199],{"class":99},"BY",[93,201,202],{"class":110}," id;\n",[26,204,205,206],{},"A watermark control table, created once:",[84,207,209],{"className":86,"code":208,"language":88,"meta":89,"style":89},"CREATE TABLE IF NOT EXISTS `my-project.warehouse.etl_watermarks` (\n  source_table STRING NOT NULL,\n  watermark_value TIMESTAMP,\n  loaded_at TIMESTAMP,\n  row_count INT64\n);\n",[15,210,211,232,241,250,259,264],{"__ignoreMap":89},[93,212,213,215,217,221,224,227,230],{"class":95,"line":96},[93,214,100],{"class":99},[93,216,103],{"class":99},[93,218,220],{"class":219},"svObZ"," IF",[93,222,223],{"class":99}," NOT",[93,225,226],{"class":99}," EXISTS",[93,228,229],{"class":106}," `my-project.warehouse.etl_watermarks`",[93,231,111],{"class":110},[93,233,234,237,239],{"class":95,"line":114},[93,235,236],{"class":110},"  source_table STRING ",[93,238,120],{"class":99},[93,240,123],{"class":110},[93,242,243,246,248],{"class":95,"line":126},[93,244,245],{"class":110},"  watermark_value ",[93,247,158],{"class":99},[93,249,123],{"class":110},[93,251,252,255,257],{"class":95,"line":132},[93,253,254],{"class":110},"  loaded_at ",[93,256,158],{"class":99},[93,258,123],{"class":110},[93,260,261],{"class":95,"line":141},[93,262,263],{"class":110},"  row_count INT64\n",[93,265,266],{"class":95,"line":152},[93,267,268],{"class":110},");\n",[26,270,271,272,275,276,278],{},"The watermark column must be ",[15,273,274],{},"timestamptz"," in Postgres and ",[15,277,158],{}," in BigQuery, set on every insert and update (a trigger or the ORM, not application goodwill). The primary key column must be unique in the source.",[26,280,281,282,285,286,289,290,293],{},"Credentials through Application Default Credentials (",[15,283,284],{},"gcloud auth application-default login"," locally, or the attached service account on Cloud Run or a VM). Grant BigQuery Data Editor (",[15,287,288],{},"roles\u002Fbigquery.dataEditor",") on the dataset and BigQuery Job User (",[15,291,292],{},"roles\u002Fbigquery.jobUser",") on the project.",[26,295,296,297,300],{},"A Postgres role with ",[15,298,299],{},"SELECT"," on the source table. Point it at a read replica if you have one.",[56,302,304],{"id":303},"usage","Usage",[11,306,307],{},"First run (no watermark yet) copies the whole table; every later run copies only what changed:",[84,309,313],{"className":310,"code":311,"language":312,"meta":89,"style":89},"language-bash shiki shiki-themes github-dark","python postgres_to_bigquery_incremental.py \\\n    --pg-dsn \"postgresql:\u002F\u002Fetl_reader@\u003Creplica-host>:5432\u002Fapp\" \\\n    --source-table public.orders \\\n    --project my-project \\\n    --dataset warehouse \\\n    --destination-table orders \\\n    --key-column id \\\n    --watermark-column updated_at \\\n    --lookback-minutes 10\n","bash",[15,314,315,327,337,347,357,367,377,387,397],{"__ignoreMap":89},[93,316,317,320,323],{"class":95,"line":96},[93,318,319],{"class":219},"python",[93,321,322],{"class":106}," postgres_to_bigquery_incremental.py",[93,324,326],{"class":325},"sDLfK"," \\\n",[93,328,329,332,335],{"class":95,"line":114},[93,330,331],{"class":325},"    --pg-dsn",[93,333,334],{"class":106}," \"postgresql:\u002F\u002Fetl_reader@\u003Creplica-host>:5432\u002Fapp\"",[93,336,326],{"class":325},[93,338,339,342,345],{"class":95,"line":126},[93,340,341],{"class":325},"    --source-table",[93,343,344],{"class":106}," public.orders",[93,346,326],{"class":325},[93,348,349,352,355],{"class":95,"line":132},[93,350,351],{"class":325},"    --project",[93,353,354],{"class":106}," my-project",[93,356,326],{"class":325},[93,358,359,362,365],{"class":95,"line":141},[93,360,361],{"class":325},"    --dataset",[93,363,364],{"class":106}," warehouse",[93,366,326],{"class":325},[93,368,369,372,375],{"class":95,"line":152},[93,370,371],{"class":325},"    --destination-table",[93,373,374],{"class":106}," orders",[93,376,326],{"class":325},[93,378,379,382,385],{"class":95,"line":163},[93,380,381],{"class":325},"    --key-column",[93,383,384],{"class":106}," id",[93,386,326],{"class":325},[93,388,389,392,395],{"class":95,"line":172},[93,390,391],{"class":325},"    --watermark-column",[93,393,394],{"class":106}," updated_at",[93,396,326],{"class":325},[93,398,399,402],{"class":95,"line":178},[93,400,401],{"class":325},"    --lookback-minutes",[93,403,404],{"class":325}," 10\n",[11,406,407,408,411,412,415,416,419,420,423,424,79],{},"Leave the password out of ",[15,409,410],{},"--pg-dsn",": command-line arguments show up in ",[15,413,414],{},"ps"," and shell history. libpq picks it up from a password file instead (",[15,417,418],{},"~\u002F.pgpass",", or the file named by ",[15,421,422],{},"PGPASSFILE",", mode ",[15,425,426],{},"0600",[11,428,429],{},"Sample output on a normal run:",[84,431,436],{"className":432,"code":434,"language":435,"meta":89},[433],"language-text","2025-12-17 09:00:01 INFO Last watermark for public.orders: 2025-12-16 23:58:02+00:00 (extracting from 2025-12-16 23:48:02+00:00)\n2025-12-17 09:00:03 INFO Extracted 7143 rows to \u002Ftmp\u002Ftmpk2v9x1.ndjson\n2025-12-17 09:00:09 INFO Loaded 7143 rows into my-project.warehouse.orders__staging\n2025-12-17 09:00:14 INFO Merge committed: 7143 staged rows, watermark now 2025-12-17 08:59:58+00:00\n","text",[15,437,434],{"__ignoreMap":89},[11,439,440],{},"And when nothing changed:",[84,442,445],{"className":443,"code":444,"language":435,"meta":89},[433],"2025-12-17 10:00:01 INFO Last watermark for public.orders: 2025-12-17 08:59:58+00:00 (extracting from 2025-12-17 08:49:58+00:00)\n2025-12-17 10:00:01 INFO No rows newer than the overlap window. Nothing to load.\n",[15,446,444],{"__ignoreMap":89},[11,448,449,450,453,454,456,457,460],{},"Schedule it with cron, a Cloud Run job, or an Airflow task, but never let two runs for the same table overlap. Both would use the same ",[15,451,452],{},"\u003Cdestination>__staging"," table, which each run replaces, so a second run can overwrite staging while the first is still between its load job and its ",[15,455,46],{},". With cron, wrap the command in ",[15,458,459],{},"flock -n",", which exits at once instead of starting a second copy while the previous run still holds the lock:",[84,462,464],{"className":310,"code":463,"language":312,"meta":89,"style":89},"# m h dom mon dow  command\n*\u002F15 * * * * flock -n \u002Fvar\u002Flock\u002Fpg-to-bq-orders.lock python \u002Fopt\u002Fetl\u002Fpostgres_to_bigquery_incremental.py --pg-dsn \"postgresql:\u002F\u002Fetl_reader@\u003Creplica-host>:5432\u002Fapp\" --source-table public.orders --project my-project --dataset warehouse --destination-table orders >> \u002Fvar\u002Flog\u002Fetl\u002Forders.log 2>&1\n",[15,465,466,472],{"__ignoreMap":89},[93,467,468],{"class":95,"line":96},[93,469,471],{"class":470},"sAwPA","# m h dom mon dow  command\n",[93,473,474,477,480,482,485,487,489,492,495,498,501,504],{"class":95,"line":114},[93,475,476],{"class":99},"*",[93,478,479],{"class":110},"\u002F15 ",[93,481,476],{"class":99},[93,483,484],{"class":99}," *",[93,486,484],{"class":99},[93,488,484],{"class":99},[93,490,491],{"class":110}," flock -n \u002Fvar\u002Flock\u002Fpg-to-bq-orders.lock python \u002Fopt\u002Fetl\u002Fpostgres_to_bigquery_incremental.py --pg-dsn ",[93,493,494],{"class":106},"\"postgresql:\u002F\u002Fetl_reader@\u003Creplica-host>:5432\u002Fapp\"",[93,496,497],{"class":110}," --source-table public.orders --project my-project --dataset warehouse --destination-table orders ",[93,499,500],{"class":99},">>",[93,502,503],{"class":110}," \u002Fvar\u002Flog\u002Fetl\u002Forders.log ",[93,505,506],{"class":99},"2>&1\n",[11,508,509,510,513],{},"Use one lock file per source table, and put it in a directory the cron user can write to. In Airflow, ",[15,511,512],{},"max_active_runs=1"," on the DAG does the same job.",[11,515,516,517,519,520,525],{},"Re-running after a failure is safe: the watermark has not moved, the overlap rows are re-staged, and the ",[15,518,46],{}," updates them in place instead of duplicating them. If the source schema can change under you, run the ",[521,522,524],"a",{"href":523},"\u002F2026\u002F01\u002F21\u002Fpython-data-warehouse-catching-schema-drift-before-an-etl-run\u002F","schema drift check"," as the step before this one.",[56,527,529],{"id":528},"script","Script",[84,531,534],{"className":532,"code":533,"language":319,"meta":89,"style":89},"language-python shiki shiki-themes github-dark","#!\u002Fusr\u002Fbin\u002Fenv python3\n\"\"\"\npostgres_to_bigquery_incremental.py\n\nIncrementally copies new and changed rows from a Postgres table into a\nBigQuery table, using a timestamp watermark column (e.g. updated_at).\n\nSteps:\n    1. Read the last watermark from \u003Cproject>.\u003Cdataset>.etl_watermarks.\n    2. Stream rows with watermark > (last watermark - lookback) out of\n       Postgres through a named (server-side) cursor into a local NDJSON file.\n    3. Load that file into \u003Cdestination>__staging with one load job\n       (WRITE_TRUNCATE, schema copied from the destination table).\n    4. In one BigQuery transaction: MERGE staging into the destination on the\n       key column, then advance the watermark. Any failure rolls back both.\n\nReads:  the Postgres source table; the destination table's schema; the\n        watermark control table.\nWrites: \u003Cdestination>__staging (replaced every run), the destination table,\n        and one row per source table in etl_watermarks.\n\"\"\"\n\nimport argparse\nimport base64\nimport datetime as dt\nimport decimal\nimport json\nimport logging\nimport re\nimport sys\nimport tempfile\nimport uuid\n\nimport psycopg2\nfrom google.cloud import bigquery\nfrom psycopg2 import sql\n\nlogging.basicConfig(level=logging.INFO, format=\"%(asctime)s %(levelname)s %(message)s\")\nlogger = logging.getLogger(\"pg_to_bq_incremental\")\n\nWATERMARK_TABLE = \"etl_watermarks\"\nIDENTIFIER = re.compile(r\"^[A-Za-z_][A-Za-z0-9_]*$\")\n\n\ndef json_default(value):\n    \"\"\"Serialize the Postgres types psycopg2 returns that json.dumps cannot.\"\"\"\n    if isinstance(value, (dt.datetime, dt.date, dt.time)):\n        return value.isoformat()\n    if isinstance(value, decimal.Decimal):\n        return str(value)  # keeps full precision for NUMERIC columns\n    if isinstance(value, uuid.UUID):\n        return str(value)\n    if isinstance(value, (bytes, memoryview)):\n        return base64.b64encode(bytes(value)).decode(\"ascii\")  # BYTES must be base64 in JSON\n    raise TypeError(f\"Cannot serialize {type(value).__name__}\")\n\n\ndef get_last_watermark(bq_client, project, dataset, source_table):\n    \"\"\"Return the stored watermark (aware datetime) for source_table, or None.\"\"\"\n    query = f\"\"\"\n        SELECT watermark_value\n        FROM `{project}.{dataset}.{WATERMARK_TABLE}`\n        WHERE source_table = @source_table\n    \"\"\"\n    job_config = bigquery.QueryJobConfig(\n        query_parameters=[bigquery.ScalarQueryParameter(\"source_table\", \"STRING\", source_table)]\n    )\n    rows = list(bq_client.query(query, job_config=job_config).result())\n    return rows[0][\"watermark_value\"] if rows else None\n\n\ndef extract_to_file(pg_dsn, source_table, columns, watermark_column, since, batch_size, handle):\n    \"\"\"Stream matching rows into handle as NDJSON. Returns the row count.\"\"\"\n    schema_name, table_name = source_table.split(\".\", 1) if \".\" in source_table else (\"public\", source_table)\n    query = sql.SQL(\"SELECT {cols} FROM {table}\").format(\n        cols=sql.SQL(\", \").join(sql.Identifier(c) for c in columns),\n        table=sql.Identifier(schema_name, table_name),\n    )\n    params = None\n    if since is not None:\n        query = query + sql.SQL(\" WHERE {wm} > %s\").format(wm=sql.Identifier(watermark_column))\n        params = (since,)\n\n    conn = psycopg2.connect(pg_dsn)\n    count = 0\n    try:\n        # A named cursor lives inside a transaction; the with block commits it.\n        with conn:\n            with conn.cursor(name=\"pg_to_bq_extract\") as cursor:\n                cursor.itersize = batch_size\n                cursor.execute(query, params)\n                for record in cursor:\n                    row = dict(zip(columns, record))\n                    handle.write(json.dumps(row, default=json_default).encode(\"utf-8\"))\n                    handle.write(b\"\\n\")\n                    count += 1\n    finally:\n        conn.close()  # leaving \"with conn\" ends the transaction, not the connection\n    return count\n\n\ndef load_staging(bq_client, handle, staging_id, schema):\n    \"\"\"Replace the staging table with the file contents using one load job.\"\"\"\n    job_config = bigquery.LoadJobConfig(\n        source_format=bigquery.SourceFormat.NEWLINE_DELIMITED_JSON,\n        write_disposition=bigquery.WriteDisposition.WRITE_TRUNCATE,\n        schema=schema,\n    )\n    load_job = bq_client.load_table_from_file(handle, staging_id, rewind=True, job_config=job_config)\n    load_job.result()  # raises google.api_core.exceptions.GoogleAPICallError on failure\n    return load_job.output_rows\n\n\ndef merge_and_advance(bq_client, dest_id, staging_id, watermark_id, columns, key, watermark_column, source_table):\n    \"\"\"MERGE staging into the destination and advance the watermark atomically.\"\"\"\n    col = lambda name: f\"`{name}`\"\n    update_set = \", \".join(f\"{col(c)} = S.{col(c)}\" for c in columns if c != key)\n    insert_cols = \", \".join(col(c) for c in columns)\n    insert_vals = \", \".join(f\"S.{col(c)}\" for c in columns)\n\n    script = f\"\"\"\n    BEGIN TRANSACTION;\n\n    MERGE `{dest_id}` AS T\n    USING (\n      SELECT * EXCEPT (_rn) FROM (\n        SELECT *, ROW_NUMBER() OVER (PARTITION BY {col(key)} ORDER BY {col(watermark_column)} DESC) AS _rn\n        FROM `{staging_id}`\n      )\n      WHERE _rn = 1\n    ) AS S\n    ON T.{col(key)} = S.{col(key)}\n    WHEN MATCHED AND S.{col(watermark_column)} >= T.{col(watermark_column)} THEN\n      UPDATE SET {update_set}\n    WHEN NOT MATCHED THEN\n      INSERT ({insert_cols}) VALUES ({insert_vals});\n\n    MERGE `{watermark_id}` AS W\n    USING (\n      SELECT @source_table AS source_table,\n             MAX({col(watermark_column)}) AS watermark_value,\n             COUNT(*) AS row_count\n      FROM `{staging_id}`\n    ) AS S\n    ON W.source_table = S.source_table\n    WHEN MATCHED THEN\n      UPDATE SET watermark_value = GREATEST(W.watermark_value, S.watermark_value),\n                 loaded_at = CURRENT_TIMESTAMP(),\n                 row_count = S.row_count\n    WHEN NOT MATCHED THEN\n      INSERT (source_table, watermark_value, loaded_at, row_count)\n      VALUES (S.source_table, S.watermark_value, CURRENT_TIMESTAMP(), S.row_count);\n\n    COMMIT TRANSACTION;\n    \"\"\"\n    job_config = bigquery.QueryJobConfig(\n        query_parameters=[bigquery.ScalarQueryParameter(\"source_table\", \"STRING\", source_table)]\n    )\n    # No exception handler: if any statement fails, BigQuery rolls the transaction back\n    # and the job (and therefore .result()) fails.\n    bq_client.query(script, job_config=job_config).result()\n\n\ndef run(args):\n    for name in (args.destination_table, args.key_column, args.watermark_column):\n        if not IDENTIFIER.match(name):\n            raise ValueError(f\"Not a plain identifier: {name!r}\")\n\n    bq_client = bigquery.Client(project=args.project)\n    dest_id = f\"{args.project}.{args.dataset}.{args.destination_table}\"\n    staging_id = f\"{dest_id}__staging\"\n    watermark_id = f\"{args.project}.{args.dataset}.{WATERMARK_TABLE}\"\n\n    destination = bq_client.get_table(dest_id)\n    columns = [field.name for field in destination.schema]\n    for required in (args.key_column, args.watermark_column):\n        if required not in columns:\n            raise ValueError(f\"Column {required!r} is not in {dest_id}\")\n\n    last_watermark = get_last_watermark(bq_client, args.project, args.dataset, args.source_table)\n    since = None\n    if last_watermark is not None:\n        since = last_watermark - dt.timedelta(minutes=args.lookback_minutes)\n    logger.info(\"Last watermark for %s: %s (extracting from %s)\", args.source_table, last_watermark, since)\n\n    with tempfile.NamedTemporaryFile(mode=\"w+b\", suffix=\".ndjson\") as handle:\n        count = extract_to_file(\n            args.pg_dsn, args.source_table, columns, args.watermark_column, since, args.batch_size, handle\n        )\n        if count == 0:\n            logger.info(\"No rows newer than the overlap window. Nothing to load.\")\n            return\n        logger.info(\"Extracted %d rows to %s\", count, handle.name)\n\n        loaded = load_staging(bq_client, handle, staging_id, destination.schema)\n        logger.info(\"Loaded %d rows into %s\", loaded, staging_id)\n\n    merge_and_advance(\n        bq_client, dest_id, staging_id, watermark_id, columns,\n        args.key_column, args.watermark_column, args.source_table,\n    )\n    new_watermark = get_last_watermark(bq_client, args.project, args.dataset, args.source_table)\n    logger.info(\"Merge committed: %d staged rows, watermark now %s\", loaded, new_watermark)\n\n\ndef parse_args():\n    parser = argparse.ArgumentParser(description=\"Incrementally load a Postgres table into BigQuery.\")\n    parser.add_argument(\"--pg-dsn\", required=True, help=\"libpq connection string or URI for the source\")\n    parser.add_argument(\"--source-table\", required=True, help=\"Postgres table, optionally schema-qualified (public.orders)\")\n    parser.add_argument(\"--project\", required=True, help=\"GCP project ID\")\n    parser.add_argument(\"--dataset\", required=True, help=\"BigQuery dataset holding the destination and etl_watermarks\")\n    parser.add_argument(\"--destination-table\", required=True, help=\"Existing BigQuery destination table\")\n    parser.add_argument(\"--key-column\", default=\"id\", help=\"Primary key used by the MERGE (default: id)\")\n    parser.add_argument(\"--watermark-column\", default=\"updated_at\", help=\"timestamptz column that changes on every write\")\n    parser.add_argument(\"--lookback-minutes\", type=int, default=10, help=\"Overlap re-read each run (default: 10)\")\n    parser.add_argument(\"--batch-size\", type=int, default=5000, help=\"Rows fetched per round trip (default: 5000)\")\n    args = parser.parse_args()\n    if args.lookback_minutes \u003C 0:\n        parser.error(\"--lookback-minutes must be 0 or more\")\n    if args.batch_size \u003C 1:\n        parser.error(\"--batch-size must be at least 1\")\n    return args\n\n\ndef main():\n    args = parse_args()\n    try:\n        run(args)\n    except Exception:\n        logger.exception(\"Incremental load failed\")\n        sys.exit(1)\n\n\nif __name__ == \"__main__\":\n    main()\n",[15,535,536,541,546,551,557,562,567,571,576,581,586,592,598,604,610,616,621,627,633,639,645,650,655,664,672,686,694,702,710,718,726,734,742,747,755,769,782,787,830,846,851,863,891,896,901,913,919,931,940,950,964,980,990,1011,1033,1064,1069,1074,1085,1091,1104,1110,1143,1149,1155,1166,1188,1194,1216,1249,1254,1259,1270,1276,1319,1346,1375,1386,1391,1401,1421,1462,1473,1478,1489,1500,1508,1514,1523,1547,1558,1564,1577,1596,1616,1634,1646,1654,1663,1671,1676,1681,1692,1698,1708,1724,1739,1750,1755,1783,1792,1800,1805,1810,1821,1827,1855,1911,1933,1966,1971,1983,1989,1994,2010,2016,2022,2048,2062,2068,2074,2080,2101,2125,2138,2144,2169,2174,2189,2194,2200,2215,2221,2235,2240,2246,2252,2258,2264,2270,2275,2281,2287,2292,2298,2303,2312,2329,2334,2340,2346,2359,2364,2369,2380,2394,2408,2437,2442,2460,2500,2521,2553,2558,2569,2590,2603,2618,2654,2659,2670,2680,2696,2720,2747,2752,2786,2797,2803,2809,2825,2836,2842,2864,2869,2880,2900,2905,2911,2917,2923,2928,2938,2958,2963,2968,2979,3000,3029,3056,3083,3110,3137,3165,3193,3231,3267,3278,3293,3304,3319,3329,3337,3342,3347,3357,3367,3374,3380,3391,3402,3412,3417,3422,3438],{"__ignoreMap":89},[93,537,538],{"class":95,"line":96},[93,539,540],{"class":470},"#!\u002Fusr\u002Fbin\u002Fenv python3\n",[93,542,543],{"class":95,"line":114},[93,544,545],{"class":106},"\"\"\"\n",[93,547,548],{"class":95,"line":126},[93,549,550],{"class":106},"postgres_to_bigquery_incremental.py\n",[93,552,553],{"class":95,"line":132},[93,554,556],{"emptyLinePlaceholder":555},true,"\n",[93,558,559],{"class":95,"line":141},[93,560,561],{"class":106},"Incrementally copies new and changed rows from a Postgres table into a\n",[93,563,564],{"class":95,"line":152},[93,565,566],{"class":106},"BigQuery table, using a timestamp watermark column (e.g. updated_at).\n",[93,568,569],{"class":95,"line":163},[93,570,556],{"emptyLinePlaceholder":555},[93,572,573],{"class":95,"line":172},[93,574,575],{"class":106},"Steps:\n",[93,577,578],{"class":95,"line":178},[93,579,580],{"class":106},"    1. Read the last watermark from \u003Cproject>.\u003Cdataset>.etl_watermarks.\n",[93,582,583],{"class":95,"line":193},[93,584,585],{"class":106},"    2. Stream rows with watermark > (last watermark - lookback) out of\n",[93,587,589],{"class":95,"line":588},11,[93,590,591],{"class":106},"       Postgres through a named (server-side) cursor into a local NDJSON file.\n",[93,593,595],{"class":95,"line":594},12,[93,596,597],{"class":106},"    3. Load that file into \u003Cdestination>__staging with one load job\n",[93,599,601],{"class":95,"line":600},13,[93,602,603],{"class":106},"       (WRITE_TRUNCATE, schema copied from the destination table).\n",[93,605,607],{"class":95,"line":606},14,[93,608,609],{"class":106},"    4. In one BigQuery transaction: MERGE staging into the destination on the\n",[93,611,613],{"class":95,"line":612},15,[93,614,615],{"class":106},"       key column, then advance the watermark. Any failure rolls back both.\n",[93,617,619],{"class":95,"line":618},16,[93,620,556],{"emptyLinePlaceholder":555},[93,622,624],{"class":95,"line":623},17,[93,625,626],{"class":106},"Reads:  the Postgres source table; the destination table's schema; the\n",[93,628,630],{"class":95,"line":629},18,[93,631,632],{"class":106},"        watermark control table.\n",[93,634,636],{"class":95,"line":635},19,[93,637,638],{"class":106},"Writes: \u003Cdestination>__staging (replaced every run), the destination table,\n",[93,640,642],{"class":95,"line":641},20,[93,643,644],{"class":106},"        and one row per source table in etl_watermarks.\n",[93,646,648],{"class":95,"line":647},21,[93,649,545],{"class":106},[93,651,653],{"class":95,"line":652},22,[93,654,556],{"emptyLinePlaceholder":555},[93,656,658,661],{"class":95,"line":657},23,[93,659,660],{"class":99},"import",[93,662,663],{"class":110}," argparse\n",[93,665,667,669],{"class":95,"line":666},24,[93,668,660],{"class":99},[93,670,671],{"class":110}," base64\n",[93,673,675,677,680,683],{"class":95,"line":674},25,[93,676,660],{"class":99},[93,678,679],{"class":110}," datetime ",[93,681,682],{"class":99},"as",[93,684,685],{"class":110}," dt\n",[93,687,689,691],{"class":95,"line":688},26,[93,690,660],{"class":99},[93,692,693],{"class":110}," decimal\n",[93,695,697,699],{"class":95,"line":696},27,[93,698,660],{"class":99},[93,700,701],{"class":110}," json\n",[93,703,705,707],{"class":95,"line":704},28,[93,706,660],{"class":99},[93,708,709],{"class":110}," logging\n",[93,711,713,715],{"class":95,"line":712},29,[93,714,660],{"class":99},[93,716,717],{"class":110}," re\n",[93,719,721,723],{"class":95,"line":720},30,[93,722,660],{"class":99},[93,724,725],{"class":110}," sys\n",[93,727,729,731],{"class":95,"line":728},31,[93,730,660],{"class":99},[93,732,733],{"class":110}," tempfile\n",[93,735,737,739],{"class":95,"line":736},32,[93,738,660],{"class":99},[93,740,741],{"class":110}," uuid\n",[93,743,745],{"class":95,"line":744},33,[93,746,556],{"emptyLinePlaceholder":555},[93,748,750,752],{"class":95,"line":749},34,[93,751,660],{"class":99},[93,753,754],{"class":110}," psycopg2\n",[93,756,758,761,764,766],{"class":95,"line":757},35,[93,759,760],{"class":99},"from",[93,762,763],{"class":110}," google.cloud ",[93,765,660],{"class":99},[93,767,768],{"class":110}," bigquery\n",[93,770,772,774,777,779],{"class":95,"line":771},36,[93,773,760],{"class":99},[93,775,776],{"class":110}," psycopg2 ",[93,778,660],{"class":99},[93,780,781],{"class":110}," sql\n",[93,783,785],{"class":95,"line":784},37,[93,786,556],{"emptyLinePlaceholder":555},[93,788,790,793,797,800,803,806,809,812,814,817,820,823,826,828],{"class":95,"line":789},38,[93,791,792],{"class":110},"logging.basicConfig(",[93,794,796],{"class":795},"s9osk","level",[93,798,799],{"class":99},"=",[93,801,802],{"class":110},"logging.",[93,804,805],{"class":325},"INFO",[93,807,808],{"class":110},", ",[93,810,811],{"class":795},"format",[93,813,799],{"class":99},[93,815,816],{"class":106},"\"",[93,818,819],{"class":325},"%(asctime)s",[93,821,822],{"class":325}," %(levelname)s",[93,824,825],{"class":325}," %(message)s",[93,827,816],{"class":106},[93,829,175],{"class":110},[93,831,833,836,838,841,844],{"class":95,"line":832},39,[93,834,835],{"class":110},"logger ",[93,837,799],{"class":99},[93,839,840],{"class":110}," logging.getLogger(",[93,842,843],{"class":106},"\"pg_to_bq_incremental\"",[93,845,175],{"class":110},[93,847,849],{"class":95,"line":848},40,[93,850,556],{"emptyLinePlaceholder":555},[93,852,854,857,860],{"class":95,"line":853},41,[93,855,856],{"class":325},"WATERMARK_TABLE",[93,858,859],{"class":99}," =",[93,861,862],{"class":106}," \"etl_watermarks\"\n",[93,864,866,869,871,874,877,879,882,884,887,889],{"class":95,"line":865},42,[93,867,868],{"class":325},"IDENTIFIER",[93,870,859],{"class":99},[93,872,873],{"class":110}," re.compile(",[93,875,876],{"class":99},"r",[93,878,816],{"class":106},[93,880,881],{"class":325},"^[A-Za-z_][A-Za-z0-9_]",[93,883,476],{"class":99},[93,885,886],{"class":325},"$",[93,888,816],{"class":106},[93,890,175],{"class":110},[93,892,894],{"class":95,"line":893},43,[93,895,556],{"emptyLinePlaceholder":555},[93,897,899],{"class":95,"line":898},44,[93,900,556],{"emptyLinePlaceholder":555},[93,902,904,907,910],{"class":95,"line":903},45,[93,905,906],{"class":99},"def",[93,908,909],{"class":219}," json_default",[93,911,912],{"class":110},"(value):\n",[93,914,916],{"class":95,"line":915},46,[93,917,918],{"class":106},"    \"\"\"Serialize the Postgres types psycopg2 returns that json.dumps cannot.\"\"\"\n",[93,920,922,925,928],{"class":95,"line":921},47,[93,923,924],{"class":99},"    if",[93,926,927],{"class":325}," isinstance",[93,929,930],{"class":110},"(value, (dt.datetime, dt.date, dt.time)):\n",[93,932,934,937],{"class":95,"line":933},48,[93,935,936],{"class":99},"        return",[93,938,939],{"class":110}," value.isoformat()\n",[93,941,943,945,947],{"class":95,"line":942},49,[93,944,924],{"class":99},[93,946,927],{"class":325},[93,948,949],{"class":110},"(value, decimal.Decimal):\n",[93,951,953,955,958,961],{"class":95,"line":952},50,[93,954,936],{"class":99},[93,956,957],{"class":325}," str",[93,959,960],{"class":110},"(value)  ",[93,962,963],{"class":470},"# keeps full precision for NUMERIC columns\n",[93,965,967,969,971,974,977],{"class":95,"line":966},51,[93,968,924],{"class":99},[93,970,927],{"class":325},[93,972,973],{"class":110},"(value, uuid.",[93,975,976],{"class":325},"UUID",[93,978,979],{"class":110},"):\n",[93,981,983,985,987],{"class":95,"line":982},52,[93,984,936],{"class":99},[93,986,957],{"class":325},[93,988,989],{"class":110},"(value)\n",[93,991,993,995,997,1000,1003,1005,1008],{"class":95,"line":992},53,[93,994,924],{"class":99},[93,996,927],{"class":325},[93,998,999],{"class":110},"(value, (",[93,1001,1002],{"class":325},"bytes",[93,1004,808],{"class":110},[93,1006,1007],{"class":325},"memoryview",[93,1009,1010],{"class":110},")):\n",[93,1012,1014,1016,1019,1021,1024,1027,1030],{"class":95,"line":1013},54,[93,1015,936],{"class":99},[93,1017,1018],{"class":110}," base64.b64encode(",[93,1020,1002],{"class":325},[93,1022,1023],{"class":110},"(value)).decode(",[93,1025,1026],{"class":106},"\"ascii\"",[93,1028,1029],{"class":110},")  ",[93,1031,1032],{"class":470},"# BYTES must be base64 in JSON\n",[93,1034,1036,1039,1042,1045,1048,1051,1054,1057,1060,1062],{"class":95,"line":1035},55,[93,1037,1038],{"class":99},"    raise",[93,1040,1041],{"class":325}," TypeError",[93,1043,1044],{"class":110},"(",[93,1046,1047],{"class":99},"f",[93,1049,1050],{"class":106},"\"Cannot serialize ",[93,1052,1053],{"class":325},"{type",[93,1055,1056],{"class":110},"(value).",[93,1058,1059],{"class":325},"__name__}",[93,1061,816],{"class":106},[93,1063,175],{"class":110},[93,1065,1067],{"class":95,"line":1066},56,[93,1068,556],{"emptyLinePlaceholder":555},[93,1070,1072],{"class":95,"line":1071},57,[93,1073,556],{"emptyLinePlaceholder":555},[93,1075,1077,1079,1082],{"class":95,"line":1076},58,[93,1078,906],{"class":99},[93,1080,1081],{"class":219}," get_last_watermark",[93,1083,1084],{"class":110},"(bq_client, project, dataset, source_table):\n",[93,1086,1088],{"class":95,"line":1087},59,[93,1089,1090],{"class":106},"    \"\"\"Return the stored watermark (aware datetime) for source_table, or None.\"\"\"\n",[93,1092,1094,1097,1099,1102],{"class":95,"line":1093},60,[93,1095,1096],{"class":110},"    query ",[93,1098,799],{"class":99},[93,1100,1101],{"class":99}," f",[93,1103,545],{"class":106},[93,1105,1107],{"class":95,"line":1106},61,[93,1108,1109],{"class":106},"        SELECT watermark_value\n",[93,1111,1113,1116,1119,1122,1125,1128,1130,1133,1135,1137,1140],{"class":95,"line":1112},62,[93,1114,1115],{"class":106},"        FROM `",[93,1117,1118],{"class":325},"{",[93,1120,1121],{"class":110},"project",[93,1123,1124],{"class":325},"}",[93,1126,1127],{"class":106},".",[93,1129,1118],{"class":325},[93,1131,1132],{"class":110},"dataset",[93,1134,1124],{"class":325},[93,1136,1127],{"class":106},[93,1138,1139],{"class":325},"{WATERMARK_TABLE}",[93,1141,1142],{"class":106},"`\n",[93,1144,1146],{"class":95,"line":1145},63,[93,1147,1148],{"class":106},"        WHERE source_table = @source_table\n",[93,1150,1152],{"class":95,"line":1151},64,[93,1153,1154],{"class":106},"    \"\"\"\n",[93,1156,1158,1161,1163],{"class":95,"line":1157},65,[93,1159,1160],{"class":110},"    job_config ",[93,1162,799],{"class":99},[93,1164,1165],{"class":110}," bigquery.QueryJobConfig(\n",[93,1167,1169,1172,1174,1177,1180,1182,1185],{"class":95,"line":1168},66,[93,1170,1171],{"class":795},"        query_parameters",[93,1173,799],{"class":99},[93,1175,1176],{"class":110},"[bigquery.ScalarQueryParameter(",[93,1178,1179],{"class":106},"\"source_table\"",[93,1181,808],{"class":110},[93,1183,1184],{"class":106},"\"STRING\"",[93,1186,1187],{"class":110},", source_table)]\n",[93,1189,1191],{"class":95,"line":1190},67,[93,1192,1193],{"class":110},"    )\n",[93,1195,1197,1200,1202,1205,1208,1211,1213],{"class":95,"line":1196},68,[93,1198,1199],{"class":110},"    rows ",[93,1201,799],{"class":99},[93,1203,1204],{"class":325}," list",[93,1206,1207],{"class":110},"(bq_client.query(query, ",[93,1209,1210],{"class":795},"job_config",[93,1212,799],{"class":99},[93,1214,1215],{"class":110},"job_config).result())\n",[93,1217,1219,1222,1225,1228,1231,1234,1237,1240,1243,1246],{"class":95,"line":1218},69,[93,1220,1221],{"class":99},"    return",[93,1223,1224],{"class":110}," rows[",[93,1226,1227],{"class":325},"0",[93,1229,1230],{"class":110},"][",[93,1232,1233],{"class":106},"\"watermark_value\"",[93,1235,1236],{"class":110},"] ",[93,1238,1239],{"class":99},"if",[93,1241,1242],{"class":110}," rows ",[93,1244,1245],{"class":99},"else",[93,1247,1248],{"class":325}," None\n",[93,1250,1252],{"class":95,"line":1251},70,[93,1253,556],{"emptyLinePlaceholder":555},[93,1255,1257],{"class":95,"line":1256},71,[93,1258,556],{"emptyLinePlaceholder":555},[93,1260,1262,1264,1267],{"class":95,"line":1261},72,[93,1263,906],{"class":99},[93,1265,1266],{"class":219}," extract_to_file",[93,1268,1269],{"class":110},"(pg_dsn, source_table, columns, watermark_column, since, batch_size, handle):\n",[93,1271,1273],{"class":95,"line":1272},73,[93,1274,1275],{"class":106},"    \"\"\"Stream matching rows into handle as NDJSON. Returns the row count.\"\"\"\n",[93,1277,1279,1282,1284,1287,1290,1292,1295,1298,1300,1303,1306,1309,1311,1313,1316],{"class":95,"line":1278},74,[93,1280,1281],{"class":110},"    schema_name, table_name ",[93,1283,799],{"class":99},[93,1285,1286],{"class":110}," source_table.split(",[93,1288,1289],{"class":106},"\".\"",[93,1291,808],{"class":110},[93,1293,1294],{"class":325},"1",[93,1296,1297],{"class":110},") ",[93,1299,1239],{"class":99},[93,1301,1302],{"class":106}," \".\"",[93,1304,1305],{"class":99}," in",[93,1307,1308],{"class":110}," source_table ",[93,1310,1245],{"class":99},[93,1312,75],{"class":110},[93,1314,1315],{"class":106},"\"public\"",[93,1317,1318],{"class":110},", source_table)\n",[93,1320,1322,1324,1326,1329,1332,1335,1338,1341,1343],{"class":95,"line":1321},75,[93,1323,1096],{"class":110},[93,1325,799],{"class":99},[93,1327,1328],{"class":110}," sql.SQL(",[93,1330,1331],{"class":106},"\"SELECT ",[93,1333,1334],{"class":325},"{cols}",[93,1336,1337],{"class":106}," FROM ",[93,1339,1340],{"class":325},"{table}",[93,1342,816],{"class":106},[93,1344,1345],{"class":110},").format(\n",[93,1347,1349,1352,1354,1357,1360,1363,1366,1369,1372],{"class":95,"line":1348},76,[93,1350,1351],{"class":795},"        cols",[93,1353,799],{"class":99},[93,1355,1356],{"class":110},"sql.SQL(",[93,1358,1359],{"class":106},"\", \"",[93,1361,1362],{"class":110},").join(sql.Identifier(c) ",[93,1364,1365],{"class":99},"for",[93,1367,1368],{"class":110}," c ",[93,1370,1371],{"class":99},"in",[93,1373,1374],{"class":110}," columns),\n",[93,1376,1378,1381,1383],{"class":95,"line":1377},77,[93,1379,1380],{"class":795},"        table",[93,1382,799],{"class":99},[93,1384,1385],{"class":110},"sql.Identifier(schema_name, table_name),\n",[93,1387,1389],{"class":95,"line":1388},78,[93,1390,1193],{"class":110},[93,1392,1394,1397,1399],{"class":95,"line":1393},79,[93,1395,1396],{"class":110},"    params ",[93,1398,799],{"class":99},[93,1400,1248],{"class":325},[93,1402,1404,1406,1409,1412,1415,1418],{"class":95,"line":1403},80,[93,1405,924],{"class":99},[93,1407,1408],{"class":110}," since ",[93,1410,1411],{"class":99},"is",[93,1413,1414],{"class":99}," not",[93,1416,1417],{"class":325}," None",[93,1419,1420],{"class":110},":\n",[93,1422,1424,1427,1429,1432,1435,1437,1440,1443,1446,1449,1451,1454,1457,1459],{"class":95,"line":1423},81,[93,1425,1426],{"class":110},"        query ",[93,1428,799],{"class":99},[93,1430,1431],{"class":110}," query ",[93,1433,1434],{"class":99},"+",[93,1436,1328],{"class":110},[93,1438,1439],{"class":106},"\" WHERE ",[93,1441,1442],{"class":325},"{wm}",[93,1444,1445],{"class":106}," > ",[93,1447,1448],{"class":325},"%s",[93,1450,816],{"class":106},[93,1452,1453],{"class":110},").format(",[93,1455,1456],{"class":795},"wm",[93,1458,799],{"class":99},[93,1460,1461],{"class":110},"sql.Identifier(watermark_column))\n",[93,1463,1465,1468,1470],{"class":95,"line":1464},82,[93,1466,1467],{"class":110},"        params ",[93,1469,799],{"class":99},[93,1471,1472],{"class":110}," (since,)\n",[93,1474,1476],{"class":95,"line":1475},83,[93,1477,556],{"emptyLinePlaceholder":555},[93,1479,1481,1484,1486],{"class":95,"line":1480},84,[93,1482,1483],{"class":110},"    conn ",[93,1485,799],{"class":99},[93,1487,1488],{"class":110}," psycopg2.connect(pg_dsn)\n",[93,1490,1492,1495,1497],{"class":95,"line":1491},85,[93,1493,1494],{"class":110},"    count ",[93,1496,799],{"class":99},[93,1498,1499],{"class":325}," 0\n",[93,1501,1503,1506],{"class":95,"line":1502},86,[93,1504,1505],{"class":99},"    try",[93,1507,1420],{"class":110},[93,1509,1511],{"class":95,"line":1510},87,[93,1512,1513],{"class":470},"        # A named cursor lives inside a transaction; the with block commits it.\n",[93,1515,1517,1520],{"class":95,"line":1516},88,[93,1518,1519],{"class":99},"        with",[93,1521,1522],{"class":110}," conn:\n",[93,1524,1526,1529,1532,1535,1537,1540,1542,1544],{"class":95,"line":1525},89,[93,1527,1528],{"class":99},"            with",[93,1530,1531],{"class":110}," conn.cursor(",[93,1533,1534],{"class":795},"name",[93,1536,799],{"class":99},[93,1538,1539],{"class":106},"\"pg_to_bq_extract\"",[93,1541,1297],{"class":110},[93,1543,682],{"class":99},[93,1545,1546],{"class":110}," cursor:\n",[93,1548,1550,1553,1555],{"class":95,"line":1549},90,[93,1551,1552],{"class":110},"                cursor.itersize ",[93,1554,799],{"class":99},[93,1556,1557],{"class":110}," batch_size\n",[93,1559,1561],{"class":95,"line":1560},91,[93,1562,1563],{"class":110},"                cursor.execute(query, params)\n",[93,1565,1567,1570,1573,1575],{"class":95,"line":1566},92,[93,1568,1569],{"class":99},"                for",[93,1571,1572],{"class":110}," record ",[93,1574,1371],{"class":99},[93,1576,1546],{"class":110},[93,1578,1580,1583,1585,1588,1590,1593],{"class":95,"line":1579},93,[93,1581,1582],{"class":110},"                    row ",[93,1584,799],{"class":99},[93,1586,1587],{"class":325}," dict",[93,1589,1044],{"class":110},[93,1591,1592],{"class":325},"zip",[93,1594,1595],{"class":110},"(columns, record))\n",[93,1597,1599,1602,1605,1607,1610,1613],{"class":95,"line":1598},94,[93,1600,1601],{"class":110},"                    handle.write(json.dumps(row, ",[93,1603,1604],{"class":795},"default",[93,1606,799],{"class":99},[93,1608,1609],{"class":110},"json_default).encode(",[93,1611,1612],{"class":106},"\"utf-8\"",[93,1614,1615],{"class":110},"))\n",[93,1617,1619,1622,1625,1627,1630,1632],{"class":95,"line":1618},95,[93,1620,1621],{"class":110},"                    handle.write(",[93,1623,1624],{"class":99},"b",[93,1626,816],{"class":106},[93,1628,1629],{"class":325},"\\n",[93,1631,816],{"class":106},[93,1633,175],{"class":110},[93,1635,1637,1640,1643],{"class":95,"line":1636},96,[93,1638,1639],{"class":110},"                    count ",[93,1641,1642],{"class":99},"+=",[93,1644,1645],{"class":325}," 1\n",[93,1647,1649,1652],{"class":95,"line":1648},97,[93,1650,1651],{"class":99},"    finally",[93,1653,1420],{"class":110},[93,1655,1657,1660],{"class":95,"line":1656},98,[93,1658,1659],{"class":110},"        conn.close()  ",[93,1661,1662],{"class":470},"# leaving \"with conn\" ends the transaction, not the connection\n",[93,1664,1666,1668],{"class":95,"line":1665},99,[93,1667,1221],{"class":99},[93,1669,1670],{"class":110}," count\n",[93,1672,1674],{"class":95,"line":1673},100,[93,1675,556],{"emptyLinePlaceholder":555},[93,1677,1679],{"class":95,"line":1678},101,[93,1680,556],{"emptyLinePlaceholder":555},[93,1682,1684,1686,1689],{"class":95,"line":1683},102,[93,1685,906],{"class":99},[93,1687,1688],{"class":219}," load_staging",[93,1690,1691],{"class":110},"(bq_client, handle, staging_id, schema):\n",[93,1693,1695],{"class":95,"line":1694},103,[93,1696,1697],{"class":106},"    \"\"\"Replace the staging table with the file contents using one load job.\"\"\"\n",[93,1699,1701,1703,1705],{"class":95,"line":1700},104,[93,1702,1160],{"class":110},[93,1704,799],{"class":99},[93,1706,1707],{"class":110}," bigquery.LoadJobConfig(\n",[93,1709,1711,1714,1716,1719,1722],{"class":95,"line":1710},105,[93,1712,1713],{"class":795},"        source_format",[93,1715,799],{"class":99},[93,1717,1718],{"class":110},"bigquery.SourceFormat.",[93,1720,1721],{"class":325},"NEWLINE_DELIMITED_JSON",[93,1723,123],{"class":110},[93,1725,1727,1730,1732,1735,1737],{"class":95,"line":1726},106,[93,1728,1729],{"class":795},"        write_disposition",[93,1731,799],{"class":99},[93,1733,1734],{"class":110},"bigquery.WriteDisposition.",[93,1736,39],{"class":325},[93,1738,123],{"class":110},[93,1740,1742,1745,1747],{"class":95,"line":1741},107,[93,1743,1744],{"class":795},"        schema",[93,1746,799],{"class":99},[93,1748,1749],{"class":110},"schema,\n",[93,1751,1753],{"class":95,"line":1752},108,[93,1754,1193],{"class":110},[93,1756,1758,1761,1763,1766,1769,1771,1774,1776,1778,1780],{"class":95,"line":1757},109,[93,1759,1760],{"class":110},"    load_job ",[93,1762,799],{"class":99},[93,1764,1765],{"class":110}," bq_client.load_table_from_file(handle, staging_id, ",[93,1767,1768],{"class":795},"rewind",[93,1770,799],{"class":99},[93,1772,1773],{"class":325},"True",[93,1775,808],{"class":110},[93,1777,1210],{"class":795},[93,1779,799],{"class":99},[93,1781,1782],{"class":110},"job_config)\n",[93,1784,1786,1789],{"class":95,"line":1785},110,[93,1787,1788],{"class":110},"    load_job.result()  ",[93,1790,1791],{"class":470},"# raises google.api_core.exceptions.GoogleAPICallError on failure\n",[93,1793,1795,1797],{"class":95,"line":1794},111,[93,1796,1221],{"class":99},[93,1798,1799],{"class":110}," load_job.output_rows\n",[93,1801,1803],{"class":95,"line":1802},112,[93,1804,556],{"emptyLinePlaceholder":555},[93,1806,1808],{"class":95,"line":1807},113,[93,1809,556],{"emptyLinePlaceholder":555},[93,1811,1813,1815,1818],{"class":95,"line":1812},114,[93,1814,906],{"class":99},[93,1816,1817],{"class":219}," merge_and_advance",[93,1819,1820],{"class":110},"(bq_client, dest_id, staging_id, watermark_id, columns, key, watermark_column, source_table):\n",[93,1822,1824],{"class":95,"line":1823},115,[93,1825,1826],{"class":106},"    \"\"\"MERGE staging into the destination and advance the watermark atomically.\"\"\"\n",[93,1828,1830,1833,1835,1838,1841,1843,1846,1848,1850,1852],{"class":95,"line":1829},116,[93,1831,1832],{"class":110},"    col ",[93,1834,799],{"class":99},[93,1836,1837],{"class":99}," lambda",[93,1839,1840],{"class":110}," name: ",[93,1842,1047],{"class":99},[93,1844,1845],{"class":106},"\"`",[93,1847,1118],{"class":325},[93,1849,1534],{"class":110},[93,1851,1124],{"class":325},[93,1853,1854],{"class":106},"`\"\n",[93,1856,1858,1861,1863,1866,1869,1871,1873,1875,1878,1880,1883,1885,1887,1889,1891,1894,1896,1898,1901,1903,1905,1908],{"class":95,"line":1857},117,[93,1859,1860],{"class":110},"    update_set ",[93,1862,799],{"class":99},[93,1864,1865],{"class":106}," \", \"",[93,1867,1868],{"class":110},".join(",[93,1870,1047],{"class":99},[93,1872,816],{"class":106},[93,1874,1118],{"class":325},[93,1876,1877],{"class":110},"col(c)",[93,1879,1124],{"class":325},[93,1881,1882],{"class":106}," = S.",[93,1884,1118],{"class":325},[93,1886,1877],{"class":110},[93,1888,1124],{"class":325},[93,1890,816],{"class":106},[93,1892,1893],{"class":99}," for",[93,1895,1368],{"class":110},[93,1897,1371],{"class":99},[93,1899,1900],{"class":110}," columns ",[93,1902,1239],{"class":99},[93,1904,1368],{"class":110},[93,1906,1907],{"class":99},"!=",[93,1909,1910],{"class":110}," key)\n",[93,1912,1914,1917,1919,1921,1924,1926,1928,1930],{"class":95,"line":1913},118,[93,1915,1916],{"class":110},"    insert_cols ",[93,1918,799],{"class":99},[93,1920,1865],{"class":106},[93,1922,1923],{"class":110},".join(col(c) ",[93,1925,1365],{"class":99},[93,1927,1368],{"class":110},[93,1929,1371],{"class":99},[93,1931,1932],{"class":110}," columns)\n",[93,1934,1936,1939,1941,1943,1945,1947,1950,1952,1954,1956,1958,1960,1962,1964],{"class":95,"line":1935},119,[93,1937,1938],{"class":110},"    insert_vals ",[93,1940,799],{"class":99},[93,1942,1865],{"class":106},[93,1944,1868],{"class":110},[93,1946,1047],{"class":99},[93,1948,1949],{"class":106},"\"S.",[93,1951,1118],{"class":325},[93,1953,1877],{"class":110},[93,1955,1124],{"class":325},[93,1957,816],{"class":106},[93,1959,1893],{"class":99},[93,1961,1368],{"class":110},[93,1963,1371],{"class":99},[93,1965,1932],{"class":110},[93,1967,1969],{"class":95,"line":1968},120,[93,1970,556],{"emptyLinePlaceholder":555},[93,1972,1974,1977,1979,1981],{"class":95,"line":1973},121,[93,1975,1976],{"class":110},"    script ",[93,1978,799],{"class":99},[93,1980,1101],{"class":99},[93,1982,545],{"class":106},[93,1984,1986],{"class":95,"line":1985},122,[93,1987,1988],{"class":106},"    BEGIN TRANSACTION;\n",[93,1990,1992],{"class":95,"line":1991},123,[93,1993,556],{"emptyLinePlaceholder":555},[93,1995,1997,2000,2002,2005,2007],{"class":95,"line":1996},124,[93,1998,1999],{"class":106},"    MERGE `",[93,2001,1118],{"class":325},[93,2003,2004],{"class":110},"dest_id",[93,2006,1124],{"class":325},[93,2008,2009],{"class":106},"` AS T\n",[93,2011,2013],{"class":95,"line":2012},125,[93,2014,2015],{"class":106},"    USING (\n",[93,2017,2019],{"class":95,"line":2018},126,[93,2020,2021],{"class":106},"      SELECT * EXCEPT (_rn) FROM (\n",[93,2023,2025,2028,2030,2033,2035,2038,2040,2043,2045],{"class":95,"line":2024},127,[93,2026,2027],{"class":106},"        SELECT *, ROW_NUMBER() OVER (PARTITION BY ",[93,2029,1118],{"class":325},[93,2031,2032],{"class":110},"col(key)",[93,2034,1124],{"class":325},[93,2036,2037],{"class":106}," ORDER BY ",[93,2039,1118],{"class":325},[93,2041,2042],{"class":110},"col(watermark_column)",[93,2044,1124],{"class":325},[93,2046,2047],{"class":106}," DESC) AS _rn\n",[93,2049,2051,2053,2055,2058,2060],{"class":95,"line":2050},128,[93,2052,1115],{"class":106},[93,2054,1118],{"class":325},[93,2056,2057],{"class":110},"staging_id",[93,2059,1124],{"class":325},[93,2061,1142],{"class":106},[93,2063,2065],{"class":95,"line":2064},129,[93,2066,2067],{"class":106},"      )\n",[93,2069,2071],{"class":95,"line":2070},130,[93,2072,2073],{"class":106},"      WHERE _rn = 1\n",[93,2075,2077],{"class":95,"line":2076},131,[93,2078,2079],{"class":106},"    ) AS S\n",[93,2081,2083,2086,2088,2090,2092,2094,2096,2098],{"class":95,"line":2082},132,[93,2084,2085],{"class":106},"    ON T.",[93,2087,1118],{"class":325},[93,2089,2032],{"class":110},[93,2091,1124],{"class":325},[93,2093,1882],{"class":106},[93,2095,1118],{"class":325},[93,2097,2032],{"class":110},[93,2099,2100],{"class":325},"}\n",[93,2102,2104,2107,2109,2111,2113,2116,2118,2120,2122],{"class":95,"line":2103},133,[93,2105,2106],{"class":106},"    WHEN MATCHED AND S.",[93,2108,1118],{"class":325},[93,2110,2042],{"class":110},[93,2112,1124],{"class":325},[93,2114,2115],{"class":106}," >= T.",[93,2117,1118],{"class":325},[93,2119,2042],{"class":110},[93,2121,1124],{"class":325},[93,2123,2124],{"class":106}," THEN\n",[93,2126,2128,2131,2133,2136],{"class":95,"line":2127},134,[93,2129,2130],{"class":106},"      UPDATE SET ",[93,2132,1118],{"class":325},[93,2134,2135],{"class":110},"update_set",[93,2137,2100],{"class":325},[93,2139,2141],{"class":95,"line":2140},135,[93,2142,2143],{"class":106},"    WHEN NOT MATCHED THEN\n",[93,2145,2147,2150,2152,2155,2157,2160,2162,2165,2167],{"class":95,"line":2146},136,[93,2148,2149],{"class":106},"      INSERT (",[93,2151,1118],{"class":325},[93,2153,2154],{"class":110},"insert_cols",[93,2156,1124],{"class":325},[93,2158,2159],{"class":106},") VALUES (",[93,2161,1118],{"class":325},[93,2163,2164],{"class":110},"insert_vals",[93,2166,1124],{"class":325},[93,2168,268],{"class":106},[93,2170,2172],{"class":95,"line":2171},137,[93,2173,556],{"emptyLinePlaceholder":555},[93,2175,2177,2179,2181,2184,2186],{"class":95,"line":2176},138,[93,2178,1999],{"class":106},[93,2180,1118],{"class":325},[93,2182,2183],{"class":110},"watermark_id",[93,2185,1124],{"class":325},[93,2187,2188],{"class":106},"` AS W\n",[93,2190,2192],{"class":95,"line":2191},139,[93,2193,2015],{"class":106},[93,2195,2197],{"class":95,"line":2196},140,[93,2198,2199],{"class":106},"      SELECT @source_table AS source_table,\n",[93,2201,2203,2206,2208,2210,2212],{"class":95,"line":2202},141,[93,2204,2205],{"class":106},"             MAX(",[93,2207,1118],{"class":325},[93,2209,2042],{"class":110},[93,2211,1124],{"class":325},[93,2213,2214],{"class":106},") AS watermark_value,\n",[93,2216,2218],{"class":95,"line":2217},142,[93,2219,2220],{"class":106},"             COUNT(*) AS row_count\n",[93,2222,2224,2227,2229,2231,2233],{"class":95,"line":2223},143,[93,2225,2226],{"class":106},"      FROM `",[93,2228,1118],{"class":325},[93,2230,2057],{"class":110},[93,2232,1124],{"class":325},[93,2234,1142],{"class":106},[93,2236,2238],{"class":95,"line":2237},144,[93,2239,2079],{"class":106},[93,2241,2243],{"class":95,"line":2242},145,[93,2244,2245],{"class":106},"    ON W.source_table = S.source_table\n",[93,2247,2249],{"class":95,"line":2248},146,[93,2250,2251],{"class":106},"    WHEN MATCHED THEN\n",[93,2253,2255],{"class":95,"line":2254},147,[93,2256,2257],{"class":106},"      UPDATE SET watermark_value = GREATEST(W.watermark_value, S.watermark_value),\n",[93,2259,2261],{"class":95,"line":2260},148,[93,2262,2263],{"class":106},"                 loaded_at = CURRENT_TIMESTAMP(),\n",[93,2265,2267],{"class":95,"line":2266},149,[93,2268,2269],{"class":106},"                 row_count = S.row_count\n",[93,2271,2273],{"class":95,"line":2272},150,[93,2274,2143],{"class":106},[93,2276,2278],{"class":95,"line":2277},151,[93,2279,2280],{"class":106},"      INSERT (source_table, watermark_value, loaded_at, row_count)\n",[93,2282,2284],{"class":95,"line":2283},152,[93,2285,2286],{"class":106},"      VALUES (S.source_table, S.watermark_value, CURRENT_TIMESTAMP(), S.row_count);\n",[93,2288,2290],{"class":95,"line":2289},153,[93,2291,556],{"emptyLinePlaceholder":555},[93,2293,2295],{"class":95,"line":2294},154,[93,2296,2297],{"class":106},"    COMMIT TRANSACTION;\n",[93,2299,2301],{"class":95,"line":2300},155,[93,2302,1154],{"class":106},[93,2304,2306,2308,2310],{"class":95,"line":2305},156,[93,2307,1160],{"class":110},[93,2309,799],{"class":99},[93,2311,1165],{"class":110},[93,2313,2315,2317,2319,2321,2323,2325,2327],{"class":95,"line":2314},157,[93,2316,1171],{"class":795},[93,2318,799],{"class":99},[93,2320,1176],{"class":110},[93,2322,1179],{"class":106},[93,2324,808],{"class":110},[93,2326,1184],{"class":106},[93,2328,1187],{"class":110},[93,2330,2332],{"class":95,"line":2331},158,[93,2333,1193],{"class":110},[93,2335,2337],{"class":95,"line":2336},159,[93,2338,2339],{"class":470},"    # No exception handler: if any statement fails, BigQuery rolls the transaction back\n",[93,2341,2343],{"class":95,"line":2342},160,[93,2344,2345],{"class":470},"    # and the job (and therefore .result()) fails.\n",[93,2347,2349,2352,2354,2356],{"class":95,"line":2348},161,[93,2350,2351],{"class":110},"    bq_client.query(script, ",[93,2353,1210],{"class":795},[93,2355,799],{"class":99},[93,2357,2358],{"class":110},"job_config).result()\n",[93,2360,2362],{"class":95,"line":2361},162,[93,2363,556],{"emptyLinePlaceholder":555},[93,2365,2367],{"class":95,"line":2366},163,[93,2368,556],{"emptyLinePlaceholder":555},[93,2370,2372,2374,2377],{"class":95,"line":2371},164,[93,2373,906],{"class":99},[93,2375,2376],{"class":219}," run",[93,2378,2379],{"class":110},"(args):\n",[93,2381,2383,2386,2389,2391],{"class":95,"line":2382},165,[93,2384,2385],{"class":99},"    for",[93,2387,2388],{"class":110}," name ",[93,2390,1371],{"class":99},[93,2392,2393],{"class":110}," (args.destination_table, args.key_column, args.watermark_column):\n",[93,2395,2397,2400,2402,2405],{"class":95,"line":2396},166,[93,2398,2399],{"class":99},"        if",[93,2401,1414],{"class":99},[93,2403,2404],{"class":325}," IDENTIFIER",[93,2406,2407],{"class":110},".match(name):\n",[93,2409,2411,2414,2417,2419,2421,2424,2426,2428,2431,2433,2435],{"class":95,"line":2410},167,[93,2412,2413],{"class":99},"            raise",[93,2415,2416],{"class":325}," ValueError",[93,2418,1044],{"class":110},[93,2420,1047],{"class":99},[93,2422,2423],{"class":106},"\"Not a plain identifier: ",[93,2425,1118],{"class":325},[93,2427,1534],{"class":110},[93,2429,2430],{"class":99},"!r",[93,2432,1124],{"class":325},[93,2434,816],{"class":106},[93,2436,175],{"class":110},[93,2438,2440],{"class":95,"line":2439},168,[93,2441,556],{"emptyLinePlaceholder":555},[93,2443,2445,2448,2450,2453,2455,2457],{"class":95,"line":2444},169,[93,2446,2447],{"class":110},"    bq_client ",[93,2449,799],{"class":99},[93,2451,2452],{"class":110}," bigquery.Client(",[93,2454,1121],{"class":795},[93,2456,799],{"class":99},[93,2458,2459],{"class":110},"args.project)\n",[93,2461,2463,2466,2468,2470,2472,2474,2477,2479,2481,2483,2486,2488,2490,2492,2495,2497],{"class":95,"line":2462},170,[93,2464,2465],{"class":110},"    dest_id ",[93,2467,799],{"class":99},[93,2469,1101],{"class":99},[93,2471,816],{"class":106},[93,2473,1118],{"class":325},[93,2475,2476],{"class":110},"args.project",[93,2478,1124],{"class":325},[93,2480,1127],{"class":106},[93,2482,1118],{"class":325},[93,2484,2485],{"class":110},"args.dataset",[93,2487,1124],{"class":325},[93,2489,1127],{"class":106},[93,2491,1118],{"class":325},[93,2493,2494],{"class":110},"args.destination_table",[93,2496,1124],{"class":325},[93,2498,2499],{"class":106},"\"\n",[93,2501,2503,2506,2508,2510,2512,2514,2516,2518],{"class":95,"line":2502},171,[93,2504,2505],{"class":110},"    staging_id ",[93,2507,799],{"class":99},[93,2509,1101],{"class":99},[93,2511,816],{"class":106},[93,2513,1118],{"class":325},[93,2515,2004],{"class":110},[93,2517,1124],{"class":325},[93,2519,2520],{"class":106},"__staging\"\n",[93,2522,2524,2527,2529,2531,2533,2535,2537,2539,2541,2543,2545,2547,2549,2551],{"class":95,"line":2523},172,[93,2525,2526],{"class":110},"    watermark_id ",[93,2528,799],{"class":99},[93,2530,1101],{"class":99},[93,2532,816],{"class":106},[93,2534,1118],{"class":325},[93,2536,2476],{"class":110},[93,2538,1124],{"class":325},[93,2540,1127],{"class":106},[93,2542,1118],{"class":325},[93,2544,2485],{"class":110},[93,2546,1124],{"class":325},[93,2548,1127],{"class":106},[93,2550,1139],{"class":325},[93,2552,2499],{"class":106},[93,2554,2556],{"class":95,"line":2555},173,[93,2557,556],{"emptyLinePlaceholder":555},[93,2559,2561,2564,2566],{"class":95,"line":2560},174,[93,2562,2563],{"class":110},"    destination ",[93,2565,799],{"class":99},[93,2567,2568],{"class":110}," bq_client.get_table(dest_id)\n",[93,2570,2572,2575,2577,2580,2582,2585,2587],{"class":95,"line":2571},175,[93,2573,2574],{"class":110},"    columns ",[93,2576,799],{"class":99},[93,2578,2579],{"class":110}," [field.name ",[93,2581,1365],{"class":99},[93,2583,2584],{"class":110}," field ",[93,2586,1371],{"class":99},[93,2588,2589],{"class":110}," destination.schema]\n",[93,2591,2593,2595,2598,2600],{"class":95,"line":2592},176,[93,2594,2385],{"class":99},[93,2596,2597],{"class":110}," required ",[93,2599,1371],{"class":99},[93,2601,2602],{"class":110}," (args.key_column, args.watermark_column):\n",[93,2604,2606,2608,2610,2613,2615],{"class":95,"line":2605},177,[93,2607,2399],{"class":99},[93,2609,2597],{"class":110},[93,2611,2612],{"class":99},"not",[93,2614,1305],{"class":99},[93,2616,2617],{"class":110}," columns:\n",[93,2619,2621,2623,2625,2627,2629,2632,2634,2637,2639,2641,2644,2646,2648,2650,2652],{"class":95,"line":2620},178,[93,2622,2413],{"class":99},[93,2624,2416],{"class":325},[93,2626,1044],{"class":110},[93,2628,1047],{"class":99},[93,2630,2631],{"class":106},"\"Column ",[93,2633,1118],{"class":325},[93,2635,2636],{"class":110},"required",[93,2638,2430],{"class":99},[93,2640,1124],{"class":325},[93,2642,2643],{"class":106}," is not in ",[93,2645,1118],{"class":325},[93,2647,2004],{"class":110},[93,2649,1124],{"class":325},[93,2651,816],{"class":106},[93,2653,175],{"class":110},[93,2655,2657],{"class":95,"line":2656},179,[93,2658,556],{"emptyLinePlaceholder":555},[93,2660,2662,2665,2667],{"class":95,"line":2661},180,[93,2663,2664],{"class":110},"    last_watermark ",[93,2666,799],{"class":99},[93,2668,2669],{"class":110}," get_last_watermark(bq_client, args.project, args.dataset, args.source_table)\n",[93,2671,2673,2676,2678],{"class":95,"line":2672},181,[93,2674,2675],{"class":110},"    since ",[93,2677,799],{"class":99},[93,2679,1248],{"class":325},[93,2681,2683,2685,2688,2690,2692,2694],{"class":95,"line":2682},182,[93,2684,924],{"class":99},[93,2686,2687],{"class":110}," last_watermark ",[93,2689,1411],{"class":99},[93,2691,1414],{"class":99},[93,2693,1417],{"class":325},[93,2695,1420],{"class":110},[93,2697,2699,2702,2704,2706,2709,2712,2715,2717],{"class":95,"line":2698},183,[93,2700,2701],{"class":110},"        since ",[93,2703,799],{"class":99},[93,2705,2687],{"class":110},[93,2707,2708],{"class":99},"-",[93,2710,2711],{"class":110}," dt.timedelta(",[93,2713,2714],{"class":795},"minutes",[93,2716,799],{"class":99},[93,2718,2719],{"class":110},"args.lookback_minutes)\n",[93,2721,2723,2726,2729,2731,2734,2736,2739,2741,2744],{"class":95,"line":2722},184,[93,2724,2725],{"class":110},"    logger.info(",[93,2727,2728],{"class":106},"\"Last watermark for ",[93,2730,1448],{"class":325},[93,2732,2733],{"class":106},": ",[93,2735,1448],{"class":325},[93,2737,2738],{"class":106}," (extracting from ",[93,2740,1448],{"class":325},[93,2742,2743],{"class":106},")\"",[93,2745,2746],{"class":110},", args.source_table, last_watermark, since)\n",[93,2748,2750],{"class":95,"line":2749},185,[93,2751,556],{"emptyLinePlaceholder":555},[93,2753,2755,2758,2761,2764,2766,2769,2771,2774,2776,2779,2781,2783],{"class":95,"line":2754},186,[93,2756,2757],{"class":99},"    with",[93,2759,2760],{"class":110}," tempfile.NamedTemporaryFile(",[93,2762,2763],{"class":795},"mode",[93,2765,799],{"class":99},[93,2767,2768],{"class":106},"\"w+b\"",[93,2770,808],{"class":110},[93,2772,2773],{"class":795},"suffix",[93,2775,799],{"class":99},[93,2777,2778],{"class":106},"\".ndjson\"",[93,2780,1297],{"class":110},[93,2782,682],{"class":99},[93,2784,2785],{"class":110}," handle:\n",[93,2787,2789,2792,2794],{"class":95,"line":2788},187,[93,2790,2791],{"class":110},"        count ",[93,2793,799],{"class":99},[93,2795,2796],{"class":110}," extract_to_file(\n",[93,2798,2800],{"class":95,"line":2799},188,[93,2801,2802],{"class":110},"            args.pg_dsn, args.source_table, columns, args.watermark_column, since, args.batch_size, handle\n",[93,2804,2806],{"class":95,"line":2805},189,[93,2807,2808],{"class":110},"        )\n",[93,2810,2812,2814,2817,2820,2823],{"class":95,"line":2811},190,[93,2813,2399],{"class":99},[93,2815,2816],{"class":110}," count ",[93,2818,2819],{"class":99},"==",[93,2821,2822],{"class":325}," 0",[93,2824,1420],{"class":110},[93,2826,2828,2831,2834],{"class":95,"line":2827},191,[93,2829,2830],{"class":110},"            logger.info(",[93,2832,2833],{"class":106},"\"No rows newer than the overlap window. Nothing to load.\"",[93,2835,175],{"class":110},[93,2837,2839],{"class":95,"line":2838},192,[93,2840,2841],{"class":99},"            return\n",[93,2843,2845,2848,2851,2854,2857,2859,2861],{"class":95,"line":2844},193,[93,2846,2847],{"class":110},"        logger.info(",[93,2849,2850],{"class":106},"\"Extracted ",[93,2852,2853],{"class":325},"%d",[93,2855,2856],{"class":106}," rows to ",[93,2858,1448],{"class":325},[93,2860,816],{"class":106},[93,2862,2863],{"class":110},", count, handle.name)\n",[93,2865,2867],{"class":95,"line":2866},194,[93,2868,556],{"emptyLinePlaceholder":555},[93,2870,2872,2875,2877],{"class":95,"line":2871},195,[93,2873,2874],{"class":110},"        loaded ",[93,2876,799],{"class":99},[93,2878,2879],{"class":110}," load_staging(bq_client, handle, staging_id, destination.schema)\n",[93,2881,2883,2885,2888,2890,2893,2895,2897],{"class":95,"line":2882},196,[93,2884,2847],{"class":110},[93,2886,2887],{"class":106},"\"Loaded ",[93,2889,2853],{"class":325},[93,2891,2892],{"class":106}," rows into ",[93,2894,1448],{"class":325},[93,2896,816],{"class":106},[93,2898,2899],{"class":110},", loaded, staging_id)\n",[93,2901,2903],{"class":95,"line":2902},197,[93,2904,556],{"emptyLinePlaceholder":555},[93,2906,2908],{"class":95,"line":2907},198,[93,2909,2910],{"class":110},"    merge_and_advance(\n",[93,2912,2914],{"class":95,"line":2913},199,[93,2915,2916],{"class":110},"        bq_client, dest_id, staging_id, watermark_id, columns,\n",[93,2918,2920],{"class":95,"line":2919},200,[93,2921,2922],{"class":110},"        args.key_column, args.watermark_column, args.source_table,\n",[93,2924,2926],{"class":95,"line":2925},201,[93,2927,1193],{"class":110},[93,2929,2931,2934,2936],{"class":95,"line":2930},202,[93,2932,2933],{"class":110},"    new_watermark ",[93,2935,799],{"class":99},[93,2937,2669],{"class":110},[93,2939,2941,2943,2946,2948,2951,2953,2955],{"class":95,"line":2940},203,[93,2942,2725],{"class":110},[93,2944,2945],{"class":106},"\"Merge committed: ",[93,2947,2853],{"class":325},[93,2949,2950],{"class":106}," staged rows, watermark now ",[93,2952,1448],{"class":325},[93,2954,816],{"class":106},[93,2956,2957],{"class":110},", loaded, new_watermark)\n",[93,2959,2961],{"class":95,"line":2960},204,[93,2962,556],{"emptyLinePlaceholder":555},[93,2964,2966],{"class":95,"line":2965},205,[93,2967,556],{"emptyLinePlaceholder":555},[93,2969,2971,2973,2976],{"class":95,"line":2970},206,[93,2972,906],{"class":99},[93,2974,2975],{"class":219}," parse_args",[93,2977,2978],{"class":110},"():\n",[93,2980,2982,2985,2987,2990,2993,2995,2998],{"class":95,"line":2981},207,[93,2983,2984],{"class":110},"    parser ",[93,2986,799],{"class":99},[93,2988,2989],{"class":110}," argparse.ArgumentParser(",[93,2991,2992],{"class":795},"description",[93,2994,799],{"class":99},[93,2996,2997],{"class":106},"\"Incrementally load a Postgres table into BigQuery.\"",[93,2999,175],{"class":110},[93,3001,3003,3006,3009,3011,3013,3015,3017,3019,3022,3024,3027],{"class":95,"line":3002},208,[93,3004,3005],{"class":110},"    parser.add_argument(",[93,3007,3008],{"class":106},"\"--pg-dsn\"",[93,3010,808],{"class":110},[93,3012,2636],{"class":795},[93,3014,799],{"class":99},[93,3016,1773],{"class":325},[93,3018,808],{"class":110},[93,3020,3021],{"class":795},"help",[93,3023,799],{"class":99},[93,3025,3026],{"class":106},"\"libpq connection string or URI for the source\"",[93,3028,175],{"class":110},[93,3030,3032,3034,3037,3039,3041,3043,3045,3047,3049,3051,3054],{"class":95,"line":3031},209,[93,3033,3005],{"class":110},[93,3035,3036],{"class":106},"\"--source-table\"",[93,3038,808],{"class":110},[93,3040,2636],{"class":795},[93,3042,799],{"class":99},[93,3044,1773],{"class":325},[93,3046,808],{"class":110},[93,3048,3021],{"class":795},[93,3050,799],{"class":99},[93,3052,3053],{"class":106},"\"Postgres table, optionally schema-qualified (public.orders)\"",[93,3055,175],{"class":110},[93,3057,3059,3061,3064,3066,3068,3070,3072,3074,3076,3078,3081],{"class":95,"line":3058},210,[93,3060,3005],{"class":110},[93,3062,3063],{"class":106},"\"--project\"",[93,3065,808],{"class":110},[93,3067,2636],{"class":795},[93,3069,799],{"class":99},[93,3071,1773],{"class":325},[93,3073,808],{"class":110},[93,3075,3021],{"class":795},[93,3077,799],{"class":99},[93,3079,3080],{"class":106},"\"GCP project ID\"",[93,3082,175],{"class":110},[93,3084,3086,3088,3091,3093,3095,3097,3099,3101,3103,3105,3108],{"class":95,"line":3085},211,[93,3087,3005],{"class":110},[93,3089,3090],{"class":106},"\"--dataset\"",[93,3092,808],{"class":110},[93,3094,2636],{"class":795},[93,3096,799],{"class":99},[93,3098,1773],{"class":325},[93,3100,808],{"class":110},[93,3102,3021],{"class":795},[93,3104,799],{"class":99},[93,3106,3107],{"class":106},"\"BigQuery dataset holding the destination and etl_watermarks\"",[93,3109,175],{"class":110},[93,3111,3113,3115,3118,3120,3122,3124,3126,3128,3130,3132,3135],{"class":95,"line":3112},212,[93,3114,3005],{"class":110},[93,3116,3117],{"class":106},"\"--destination-table\"",[93,3119,808],{"class":110},[93,3121,2636],{"class":795},[93,3123,799],{"class":99},[93,3125,1773],{"class":325},[93,3127,808],{"class":110},[93,3129,3021],{"class":795},[93,3131,799],{"class":99},[93,3133,3134],{"class":106},"\"Existing BigQuery destination table\"",[93,3136,175],{"class":110},[93,3138,3140,3142,3145,3147,3149,3151,3154,3156,3158,3160,3163],{"class":95,"line":3139},213,[93,3141,3005],{"class":110},[93,3143,3144],{"class":106},"\"--key-column\"",[93,3146,808],{"class":110},[93,3148,1604],{"class":795},[93,3150,799],{"class":99},[93,3152,3153],{"class":106},"\"id\"",[93,3155,808],{"class":110},[93,3157,3021],{"class":795},[93,3159,799],{"class":99},[93,3161,3162],{"class":106},"\"Primary key used by the MERGE (default: id)\"",[93,3164,175],{"class":110},[93,3166,3168,3170,3173,3175,3177,3179,3182,3184,3186,3188,3191],{"class":95,"line":3167},214,[93,3169,3005],{"class":110},[93,3171,3172],{"class":106},"\"--watermark-column\"",[93,3174,808],{"class":110},[93,3176,1604],{"class":795},[93,3178,799],{"class":99},[93,3180,3181],{"class":106},"\"updated_at\"",[93,3183,808],{"class":110},[93,3185,3021],{"class":795},[93,3187,799],{"class":99},[93,3189,3190],{"class":106},"\"timestamptz column that changes on every write\"",[93,3192,175],{"class":110},[93,3194,3196,3198,3201,3203,3206,3208,3211,3213,3215,3217,3220,3222,3224,3226,3229],{"class":95,"line":3195},215,[93,3197,3005],{"class":110},[93,3199,3200],{"class":106},"\"--lookback-minutes\"",[93,3202,808],{"class":110},[93,3204,3205],{"class":795},"type",[93,3207,799],{"class":99},[93,3209,3210],{"class":325},"int",[93,3212,808],{"class":110},[93,3214,1604],{"class":795},[93,3216,799],{"class":99},[93,3218,3219],{"class":325},"10",[93,3221,808],{"class":110},[93,3223,3021],{"class":795},[93,3225,799],{"class":99},[93,3227,3228],{"class":106},"\"Overlap re-read each run (default: 10)\"",[93,3230,175],{"class":110},[93,3232,3234,3236,3239,3241,3243,3245,3247,3249,3251,3253,3256,3258,3260,3262,3265],{"class":95,"line":3233},216,[93,3235,3005],{"class":110},[93,3237,3238],{"class":106},"\"--batch-size\"",[93,3240,808],{"class":110},[93,3242,3205],{"class":795},[93,3244,799],{"class":99},[93,3246,3210],{"class":325},[93,3248,808],{"class":110},[93,3250,1604],{"class":795},[93,3252,799],{"class":99},[93,3254,3255],{"class":325},"5000",[93,3257,808],{"class":110},[93,3259,3021],{"class":795},[93,3261,799],{"class":99},[93,3263,3264],{"class":106},"\"Rows fetched per round trip (default: 5000)\"",[93,3266,175],{"class":110},[93,3268,3270,3273,3275],{"class":95,"line":3269},217,[93,3271,3272],{"class":110},"    args ",[93,3274,799],{"class":99},[93,3276,3277],{"class":110}," parser.parse_args()\n",[93,3279,3281,3283,3286,3289,3291],{"class":95,"line":3280},218,[93,3282,924],{"class":99},[93,3284,3285],{"class":110}," args.lookback_minutes ",[93,3287,3288],{"class":99},"\u003C",[93,3290,2822],{"class":325},[93,3292,1420],{"class":110},[93,3294,3296,3299,3302],{"class":95,"line":3295},219,[93,3297,3298],{"class":110},"        parser.error(",[93,3300,3301],{"class":106},"\"--lookback-minutes must be 0 or more\"",[93,3303,175],{"class":110},[93,3305,3307,3309,3312,3314,3317],{"class":95,"line":3306},220,[93,3308,924],{"class":99},[93,3310,3311],{"class":110}," args.batch_size ",[93,3313,3288],{"class":99},[93,3315,3316],{"class":325}," 1",[93,3318,1420],{"class":110},[93,3320,3322,3324,3327],{"class":95,"line":3321},221,[93,3323,3298],{"class":110},[93,3325,3326],{"class":106},"\"--batch-size must be at least 1\"",[93,3328,175],{"class":110},[93,3330,3332,3334],{"class":95,"line":3331},222,[93,3333,1221],{"class":99},[93,3335,3336],{"class":110}," args\n",[93,3338,3340],{"class":95,"line":3339},223,[93,3341,556],{"emptyLinePlaceholder":555},[93,3343,3345],{"class":95,"line":3344},224,[93,3346,556],{"emptyLinePlaceholder":555},[93,3348,3350,3352,3355],{"class":95,"line":3349},225,[93,3351,906],{"class":99},[93,3353,3354],{"class":219}," main",[93,3356,2978],{"class":110},[93,3358,3360,3362,3364],{"class":95,"line":3359},226,[93,3361,3272],{"class":110},[93,3363,799],{"class":99},[93,3365,3366],{"class":110}," parse_args()\n",[93,3368,3370,3372],{"class":95,"line":3369},227,[93,3371,1505],{"class":99},[93,3373,1420],{"class":110},[93,3375,3377],{"class":95,"line":3376},228,[93,3378,3379],{"class":110},"        run(args)\n",[93,3381,3383,3386,3389],{"class":95,"line":3382},229,[93,3384,3385],{"class":99},"    except",[93,3387,3388],{"class":325}," Exception",[93,3390,1420],{"class":110},[93,3392,3394,3397,3400],{"class":95,"line":3393},230,[93,3395,3396],{"class":110},"        logger.exception(",[93,3398,3399],{"class":106},"\"Incremental load failed\"",[93,3401,175],{"class":110},[93,3403,3405,3408,3410],{"class":95,"line":3404},231,[93,3406,3407],{"class":110},"        sys.exit(",[93,3409,1294],{"class":325},[93,3411,175],{"class":110},[93,3413,3415],{"class":95,"line":3414},232,[93,3416,556],{"emptyLinePlaceholder":555},[93,3418,3420],{"class":95,"line":3419},233,[93,3421,556],{"emptyLinePlaceholder":555},[93,3423,3425,3427,3430,3433,3436],{"class":95,"line":3424},234,[93,3426,1239],{"class":99},[93,3428,3429],{"class":325}," __name__",[93,3431,3432],{"class":99}," ==",[93,3434,3435],{"class":106}," \"__main__\"",[93,3437,1420],{"class":110},[93,3439,3441],{"class":95,"line":3440},235,[93,3442,3443],{"class":110},"    main()\n",[56,3445,3447],{"id":3446},"notes","Notes",[61,3449,3450,3456,3491,3516,3534,3549,3565],{},[26,3451,3452,3455],{},[33,3453,3454],{},"Why one load job per run, not per batch."," BigQuery allows 1,500 load jobs per table per day, and failed load jobs count toward it. A job that appends every 5,000-row batch separately can burn through that on a big first load or a frequent schedule. Writing one NDJSON file and loading it once keeps each run at one load job plus one query job.",[26,3457,3458,3463,3464,3466,3467,3470,3471,808,3474,808,3477,3479,3480,3483,3484,3486,3487,3490],{},[33,3459,3460,3461,1127],{},"Why the watermark is written with DML, not ",[15,3462,53],{}," ",[15,3465,53],{}," uses the streaming API Google now calls the Storage Write API (REST), formerly ",[15,3468,3469],{},"tabledata.insertAll",". Rows written that way can't be changed by ",[15,3472,3473],{},"UPDATE",[15,3475,3476],{},"DELETE",[15,3478,46],{},", or ",[15,3481,3482],{},"TRUNCATE"," for 30 minutes, and each insert is a separate operation from the data load, so a crash between the two left them out of step. Putting both ",[15,3485,46],{}," statements in one ",[15,3488,3489],{},"BEGIN TRANSACTION ... COMMIT TRANSACTION"," block means BigQuery rolls both back if either fails.",[26,3492,3493,3496,3497,3499,3500,3503,3504,3508,3509,3511,3512,3515],{},[33,3494,3495],{},"Why the overlap window."," If ",[15,3498,17],{}," defaults to ",[15,3501,3502],{},"now()",", it holds the start time of the writing transaction, but the row only becomes visible when that transaction commits. A long transaction can commit a row stamped ",[3505,3506,3507],"em",{},"earlier"," than a watermark you already stored. Re-reading the last few minutes on every run picks those rows up, and the ",[15,3510,46],{}," makes the re-read harmless. Set ",[15,3513,3514],{},"--lookback-minutes"," longer than your longest write transaction.",[26,3517,3518,3521,3522,3525,3526,3529,3530,3533],{},[33,3519,3520],{},"Duplicates in staging."," If the overlap or a retry stages the same key twice, BigQuery raises ",[15,3523,3524],{},"UPDATE\u002FMERGE must match at most one source row for each target row",". The ",[15,3527,3528],{},"ROW_NUMBER()"," subquery keeps only the newest version of each key, and the ",[15,3531,3532],{},"S.updated_at >= T.updated_at"," condition stops an older staged copy from overwriting a newer row.",[26,3535,3536,3539,3540,3542,3543,3545,3546,3548],{},[33,3537,3538],{},"Cost."," Batch loading from local files is free by default, using a shared slot pool with no capacity guarantee. The ",[15,3541,46],{}," is billed as a query. On on-demand pricing (first 1 TiB per month free, then $6.25 per TiB in US regions when I checked), a ",[15,3544,46],{}," with an ",[15,3547,3473],{}," clause is billed for the bytes it reads plus the size of the target, or only the target partitions it touches if the table is partitioned. Partitioning the destination on a date column that recent changes cluster around keeps that second term small.",[26,3550,3551,3554,3555,3558,3559,3561,3562,1127],{},[33,3552,3553],{},"Deletes are not handled."," A hard-deleted Postgres row stays in BigQuery. Either soft-delete in the source (a ",[15,3556,3557],{},"deleted_at"," column that bumps ",[15,3560,17],{},"), or run a periodic reconciliation that loads the full list of live keys and deletes the rest with ",[15,3563,3564],{},"WHEN NOT MATCHED BY SOURCE THEN DELETE",[26,3566,3567,3570],{},[33,3568,3569],{},"When to stop polling."," Past a certain change rate, a watermark poller is the wrong tool. Google's Datastream supports PostgreSQL as a source and replicates changes into BigQuery continuously.",[56,3572,3574],{"id":3573},"references","References",[61,3576,3577,3585,3592,3599,3606,3613,3623,3630,3637,3644],{},[26,3578,3579],{},[521,3580,3584],{"href":3581,"rel":3582},"https:\u002F\u002Fcloud.google.com\u002Fbigquery\u002Fdocs\u002Freference\u002Fstandard-sql\u002Fdml-syntax",[3583],"nofollow","BigQuery DML syntax: MERGE and the \"at most one source row\" rule",[26,3586,3587],{},[521,3588,3591],{"href":3589,"rel":3590},"https:\u002F\u002Fcloud.google.com\u002Fbigquery\u002Fdocs\u002Ftransactions",[3583],"BigQuery multi-statement transactions",[26,3593,3594],{},[521,3595,3598],{"href":3596,"rel":3597},"https:\u002F\u002Fcloud.google.com\u002Fbigquery\u002Fdocs\u002Fdata-manipulation-language",[3583],"BigQuery DML: limits on rows written by the Storage Write API (REST)",[26,3600,3601],{},[521,3602,3605],{"href":3603,"rel":3604},"https:\u002F\u002Fcloud.google.com\u002Fbigquery\u002Fquotas",[3583],"BigQuery quotas and limits: load jobs",[26,3607,3608],{},[521,3609,3612],{"href":3610,"rel":3611},"https:\u002F\u002Fcloud.google.com\u002Fbigquery\u002Fpricing",[3583],"BigQuery pricing: on-demand compute, data ingestion",[26,3614,3615],{},[521,3616,3619,3620],{"href":3617,"rel":3618},"https:\u002F\u002Fcloud.google.com\u002Fpython\u002Fdocs\u002Freference\u002Fbigquery\u002Flatest\u002Fgoogle.cloud.bigquery.client.Client",[3583],"google-cloud-bigquery ",[15,3621,3622],{},"Client.load_table_from_file",[26,3624,3625],{},[521,3626,3629],{"href":3627,"rel":3628},"https:\u002F\u002Fcloud.google.com\u002Fbigquery\u002Fdocs\u002Fpartitioned-tables",[3583],"BigQuery partitioned tables",[26,3631,3632],{},[521,3633,3636],{"href":3634,"rel":3635},"https:\u002F\u002Fwww.psycopg.org\u002Fdocs\u002Fusage.html",[3583],"psycopg2: server-side cursors and connection context managers",[26,3638,3639],{},[521,3640,3643],{"href":3641,"rel":3642},"https:\u002F\u002Fwww.psycopg.org\u002Fdocs\u002Fsql.html",[3583],"psycopg2.sql: composing queries with identifiers",[26,3645,3646],{},[521,3647,3650],{"href":3648,"rel":3649},"https:\u002F\u002Fcloud.google.com\u002Fdatastream\u002Fdocs\u002Foverview",[3583],"Datastream overview",[3652,3653,3654],"style",{},"html pre.shiki code .snl16, html code.shiki .snl16{--shiki-default:#F97583}html pre.shiki code .sU2Wk, html code.shiki .sU2Wk{--shiki-default:#9ECBFF}html pre.shiki code .s95oV, html code.shiki .s95oV{--shiki-default:#E1E4E8}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html pre.shiki code .svObZ, html code.shiki .svObZ{--shiki-default:#B392F0}html pre.shiki code .sDLfK, html code.shiki .sDLfK{--shiki-default:#79B8FF}html pre.shiki code .sAwPA, html code.shiki .sAwPA{--shiki-default:#6A737D}html pre.shiki code .s9osk, html code.shiki .s9osk{--shiki-default:#FFAB70}",{"title":89,"searchDepth":114,"depth":114,"links":3656},[3657,3658,3659,3660,3661],{"id":58,"depth":114,"text":59},{"id":303,"depth":114,"text":304},{"id":528,"depth":114,"text":529},{"id":3446,"depth":114,"text":3447},{"id":3573,"depth":114,"text":3574},"techcolumnist",[3664,3665],"scripts","engineering","2025-12-17T14:00:00Z","A Python script that copies new and changed Postgres rows into BigQuery with one load job and a MERGE, advancing the watermark in the same transaction.","md",false,null,{},"\u002Fblog\u002F2025\u002F12\u002F17\u002Fpython-data-warehouse-incremental-loads-from-postgres-to-bigquery",{"title":6,"description":3667},[3662],"blog\u002F2025\u002F12\u002F17\u002Fpython-data-warehouse-incremental-loads-from-postgres-to-bigquery",[319,3677,3678,3679,88],"data-warehouse","etl","gcloud","\u002F2025\u002F12\u002F17\u002Fpython-data-warehouse-incremental-loads-from-postgres-to-bigquery\u002F","A6mo8gJ3qiTlDAwGmaj4ONcykd5JT87JgO1Fjf7WR1Q",{"title":3683,"description":3684,"date":3685,"url":3686,"categories":3687,"tags":3688,"image":3670,"readingTime":141,"canonical":3662,"sites":3690,"series":3670,"seriesOrder":3670},"Python: Azure – Cost Anomaly Alerts from the Cost Management API","A Python script that pulls daily Azure costs from the Cost Management Query API and posts a webhook alert when a day breaks from its rolling baseline.","2025-12-24T14:00:00Z","\u002F2025\u002F12\u002F24\u002Fpython-azure-automate-cost-anomaly-alerts-with-the-cost-management-api\u002F",[3664,3665],[319,3689],"azure",[3662],{"title":3692,"description":3693,"date":3694,"url":3695,"categories":3696,"tags":3697,"image":3670,"readingTime":126,"canonical":3662,"sites":3700,"series":3670,"seriesOrder":3670},"Python: Discovery – Inventory VMware Hosts and VMs via pyVmomi","A pyVmomi script that pulls ESXi host and VM inventory from vCenter with one PropertyCollector query per type and writes two CSVs for a discovery pass.","2025-12-10T14:00:00Z","\u002F2025\u002F12\u002F10\u002Fpython-discovery-inventory-vmware-hosts-and-vms-via-pyvmomi\u002F",[3664,3665],[319,3698,3699],"vmware","discovery",[3662],[3702,3709,3718],{"title":3703,"description":3704,"date":3705,"url":523,"categories":3706,"tags":3707,"image":3670,"readingTime":126,"canonical":3662,"sites":3708,"series":3670,"seriesOrder":3670},"Python: Data Warehouse – Catching Schema Drift Before an ETL Run","A Python pre-flight check that snapshots Postgres column definitions from information_schema and fails the ETL run when a column is dropped, retyped, or resized.","2026-01-21T14:00:00Z",[3664,3665],[319,3677,3678,88],[3662],{"title":3710,"description":3711,"date":3712,"url":3713,"categories":3714,"tags":3715,"image":3670,"readingTime":141,"canonical":3662,"sites":3717,"series":3670,"seriesOrder":3670},"Node.js: Data Warehouse – Streaming ETL Rows Without Buffering Them","A Node.js script that streams Postgres rows through a cursor, a transform and COPY FROM STDIN with backpressure, so memory tracks batch size, not table size.","2026-02-25T14:00:00Z","\u002F2026\u002F02\u002F25\u002Fnode-js-data-warehouse-streaming-etl-rows-without-loading-them-all-in-memory\u002F",[3664,3665],[3716,3677,3678,88],"nodejs",[3662],{"title":3719,"description":3720,"date":3721,"url":3722,"categories":3723,"tags":3725,"image":3670,"readingTime":172,"canonical":3662,"sites":3726,"series":3670,"seriesOrder":3670},"Data Warehouse: When to Stop Querying Production and Build a Warehouse","How to measure reporting load on a production Postgres database, the cheap fixes to try first, and the signals that mean a separate warehouse is finally justified.","2026-05-27T14:00:00Z","\u002F2026\u002F05\u002F27\u002Fdata-warehouse-when-to-stop-querying-production-and-build-a-warehouse\u002F",[3665,3724],"strategy",[3677,88,319],[3662],{"doc":3670,"posts":3728},[],1790052513250]