#!/usr/bin/env python3 """gem15_fuse.py - patch Google's GEM-15 bare-earth model with open data you may use commercially. Wovoka Advisory, 16 November 2026. Method and test results: https://www.wovokagroup.com/perspectives/gem15-ground-under-the-trees What it does, for one box inside one GEM-15 1 x 1 degree cell: 1. Removes the 1-2 m steps between GEM-15's image patches: the median of (GEM-15 DTM - Copernicus GLO-30) on open ground in 450 m blocks, filled, smoothed, feathered, subtracted. 2. Learns how far Copernicus (an X-band radar surface) sits below the treetops: k in Copernicus - ground = k x canopy height, fitted where GEM-15's own ground looks sound under trees. 3. Under tall canopy, and wherever GEM-15's ground sat near the treetops, uses Copernicus - k x canopy height instead, blended by canopy height (GEM-15 only under 5 m, the fallback only over 15 m). Canopy height comes from Meta and WRI's 1 m map (Tolan et al. 2024), averaged to 15 m. Every input is CC BY 4.0 or the Copernicus WorldDEM-30 licence, so the output can go on commercial maps with the credits below. (FABDEM, which Google's own app falls back to, is CC BY-NC-SA 4.0.) Writes, on GEM-15's 15 m UTM grid: _ground.tif (fused bare ground, m), _canopy.tif (GEM-15 surface minus that ground, m), _source.tif (1 GEM-15, 2 blend, 3 fallback, 4 replaced where GEM-15 collapsed, 5 Copernicus only) and _report.json (k, collapse share, a held-out test of the fallback). Usage: python3 gem15_fuse.py WEST SOUTH EAST NORTH OUT_PREFIX e.g. python3 gem15_fuse.py 125.20 11.60 125.45 11.85 samar Needs: pip install rasterio numpy scipy (reads public cloud files over HTTP; a 25 x 28 km box takes a few minutes) Limits: the box must sit in one GEM-15 cell (whole degrees). Tested in the tropics (the Philippines). Under tall forest the ground, and so the canopy height, largely repeats the canopy-height map: this removes artefacts, it does not measure anything new. Meta's imagery is mostly 2018-2020 and GEM-15's 2022-2024. Check against your own plots. Credits to show with any output: GEM-15: "This dataset is produced by Google" (CC BY 4.0; Batchu et al. 2026, doi:10.31223/X55N75). Canopy height (c) Meta and World Resources Institute (CC BY 4.0). "produced using Copernicus WorldDEM-30 (c) DLR e.V. 2010-2014 and (c) Airbus Defence and Space GmbH 2014-2018 provided under COPERNICUS by the European Union and ESA; all rights reserved" Licence of this script: MIT. No warranty. """ import json import math import sys import numpy as np import rasterio from rasterio.enums import Resampling from rasterio.warp import reproject, transform_bounds from rasterio.windows import from_bounds from scipy import ndimage as nd GEM = "https://storage.googleapis.com/climate-and-sustainability/gem-15/{grp}/w{lon}s{lat}/{kind}/zall/mosaic.tif" COP = "https://copernicus-dem-30m.s3.amazonaws.com/Copernicus_DSM_COG_10_{ns}{la:02d}_00_{ew}{lo:03d}_00_DEM/Copernicus_DSM_COG_10_{ns}{la:02d}_00_{ew}{lo:03d}_00_DEM.tif" META = "https://dataforgood-fb-data.s3.amazonaws.com/forests/v1/alsgedi_global_v6_float/chm/{qk}.tif" B = 30 # 450 m blocks def gem_cell(w, s): # GEM-15 names cells by their south-west corner; northern-hemisphere example: p10_p120/w125s11 for 125-126 E, 11-12 N. # Check the STAC catalogue for other hemispheres: storage.googleapis.com/climate-and-sustainability/gem-15/stac/catalog.json lon, lat = int(math.floor(w)), int(math.floor(s)) # Verified only for 0-20 N, 120-130 E (p0_p120, p10_p120); if a URL 404s elsewhere, look the cell up in the STAC. return f"p{lat // 10 * 10}_p{lon // 10 * 10}", lon, lat def quadkeys(box, z=9): def qk(lon, lat): n = 2 ** z x = int((lon + 180) / 360 * n) y = int((1 - math.asinh(math.tan(math.radians(lat))) / math.pi) / 2 * n) return "".join(str(((x >> (i - 1)) & 1) + 2 * ((y >> (i - 1)) & 1)) for i in range(z, 0, -1)) w, s, e, n = box st = 360 / 2 ** z / 4 return sorted({qk(x, y) for x in np.arange(w, e + st, st).clip(w, e) for y in np.arange(s, n + st, st).clip(s, n)}) def warp(url, shape, tr, crs, how): out = np.full(shape, np.nan, "f4") try: with rasterio.open(url) as r: reproject(rasterio.band(r, 1), out, dst_transform=tr, dst_crs=crs, resampling=how, dst_nodata=np.nan) except rasterio.errors.RasterioIOError: return None return out def offsets(diff, mask): H, W = diff.shape H2, W2 = H // B * B, W // B * B r = np.where(mask, diff, np.nan)[:H2, :W2].reshape(H2 // B, B, W2 // B, B).swapaxes(1, 2).reshape(H2 // B, W2 // B, -1) cnt = (~np.isnan(r)).sum(2) med = np.full(cnt.shape, np.nan) ok = cnt >= 25 if not ok.any(): return np.zeros(diff.shape, "f4") med[ok] = np.nanmedian(r[ok], axis=1) idx = nd.distance_transform_edt(np.isnan(med), return_distances=False, return_indices=True) full = np.pad(np.kron(nd.median_filter(med[tuple(idx)], size=3), np.ones((B, B))), ((0, H - H2), (0, W - W2)), mode="edge") return nd.uniform_filter(full, size=B).astype("f4") def run(box): """Fuse one box. Returns the arrays (15 m UTM grid), its transform and CRS, and the report.""" w, s, e, n = box grp, lon, lat = gem_cell(w, s) with rasterio.open(GEM.format(grp=grp, lon=lon, lat=lat, kind="dsm")) as r: win = from_bounds(*transform_bounds("EPSG:4326", r.crs, *box), r.transform).round_offsets().round_lengths() dsm = r.read(1, window=win, boundless=True, fill_value=np.nan, out_dtype="f4") # boundless keeps the grid true tr, crs = r.window_transform(win), r.crs with rasterio.open(GEM.format(grp=grp, lon=lon, lat=lat, kind="dtm")) as r: dtm = r.read(1, window=win, boundless=True, fill_value=np.nan, out_dtype="f4") cop = np.full(dsm.shape, np.nan, "f4") for la in range(int(math.floor(s)), int(math.floor(n)) + 1): for lo in range(int(math.floor(w)), int(math.floor(e)) + 1): t = warp(COP.format(ns="N" if la >= 0 else "S", la=abs(la), ew="E" if lo >= 0 else "W", lo=abs(lo)), dsm.shape, tr, crs, Resampling.bilinear) t = np.zeros(dsm.shape, "f4") if t is None else t # no tile = open sea cop = np.where(np.isnan(cop), t, cop) meta = np.full(dsm.shape, np.nan, "f4") for q in quadkeys(box): t = warp(META.format(qk=q), dsm.shape, tr, crs, Resampling.average) if t is not None: meta = np.where(np.isnan(meta), t, meta) land = ~np.isnan(cop) & (cop > 1) & ~np.isnan(dtm) & ~np.isnan(meta) chm, m0 = dsm - dtm, np.nan_to_num(meta) dtm_c = dtm - offsets(dtm - cop, land & (chm < 3) & (meta < 2)) collapse = land & (meta > 10) & (chm < 0.3 * meta) good = land & (meta > 5) & (chm >= 0.6 * meta) & (chm <= 1.4 * meta) with np.errstate(divide="ignore", invalid="ignore"): k = float(np.median(((cop - dtm_c) / meta)[good])) wgt = nd.uniform_filter(np.where(collapse, 0, np.clip((15 - m0) / 10, 0, 1)), size=3) ground = np.where(np.isnan(dtm_c), cop, wgt * dtm_c + (1 - wgt) * (cop - k * m0)) src = np.full(dsm.shape, 2, "u1") src[wgt >= 0.95], src[wgt <= 0.05], src[collapse], src[np.isnan(dtm)] = 1, 3, 4, 5 # held-out test: k from half the blocks, error on the other half where GEM-15's ground looks sound under tall trees by, bx = np.indices(dsm.shape) cb = ((by // B) + (bx // B)) % 2 == 0 tg = land & (meta > 10) & (chm >= 0.6 * meta) & (chm <= 1.4 * meta) with np.errstate(divide="ignore", invalid="ignore"): kA = float(np.median(((cop - dtm_c) / meta)[tg & cb])) test = tg & ~cb err = lambda p: round(float(np.median(np.abs(p[test] - dtm_c[test]))), 2) if test.any() else None # noqa: E731 report = {"box": box, "k": round(k, 3), "collapse_pct_of_canopy_over_10m": round(100 * float(collapse.sum() / max(1, (land & (meta > 10)).sum())), 1), "heldout_median_error_m": {"copernicus_minus_k_canopy": err(cop - kA * m0), "copernicus_as_ground": err(cop), "copernicus_minus_canopy": err(cop - m0)}} return dict(dsm=dsm, dtm=dtm, cop=cop, meta=meta, ground=ground, src=src, tr=tr, crs=crs, report=report) def main(): out = sys.argv[5] r = run(tuple(map(float, sys.argv[1:5]))) dsm, ground, src, tr, crs, report = r["dsm"], r["ground"], r["src"], r["tr"], r["crs"], r["report"] prof = dict(driver="GTiff", count=1, crs=crs, transform=tr, height=dsm.shape[0], width=dsm.shape[1], compress="deflate") for name, arr, dt, nod in (("ground", ground, "float32", np.nan), ("canopy", dsm - ground, "float32", np.nan), ("source", src, "uint8", 0)): with rasterio.open(f"{out}_{name}.tif", "w", dtype=dt, nodata=nod, **prof) as o: o.write(arr.astype(dt), 1) json.dump(report, open(f"{out}_report.json", "w"), indent=1) print(json.dumps(report, indent=1)) if __name__ == "__main__": main()