Skip to content

Instantly share code, notes, and snippets.

@vincentsarago

vincentsarago/in-memory.txt

Created April 12, 2019 14:13
Show Gist options
  • Save vincentsarago/ec5efa73660f4a739f89aaaac8a8ec92 to your computer and use it in GitHub Desktop.
Save vincentsarago/ec5efa73660f4a739f89aaaac8a8ec92 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
in-memory.txt
# IN MEMORY
Line # Mem usage Increment Line Contents
================================================
29 102.769531 MiB 102.769531 MiB @profile(precision=6)
30 def cog_translate(
31 src_path,
32 dst_path,
33 dst_kwargs,
34 indexes=None,
35 nodata=None,
36 add_mask=None,
37 overview_level=None,
38 overview_resampling="nearest",
39 in_memory=None,
40 config=None,
41 quiet=False,
42 ):
43 """
44 Create Cloud Optimized Geotiff.
45
46 Parameters
47 ----------
48 src_path : str or PathLike object
49 A dataset path or URL. Will be opened in "r" mode.
50 dst_path : str or Path-like object
51 An output dataset path or or PathLike object.
52 Will be opened in "w" mode.
53 dst_kwargs: dict
54 output dataset creation options.
55 indexes : tuple, int, optional
56 Raster band indexes to copy.
57 nodata, int, optional
58 Overwrite nodata masking values for input dataset.
59 add_mask, bool, optional
60 Force output dataset creation with a mask.
61 overview_level : int, optional (default: 6)
62 COGEO overview (decimation) level
63 overview_resampling : str, optional (default: "nearest")
64 Resampling algorithm for overviews
65 in_memory: bool, optional
66 Force processing raster in memory (default: process in memory if smal)
67 config : dict
68 Rasterio Env options.
69 quiet: bool, optional (default: False)
70 Mask processing steps.
71
72 """
73 102.769531 MiB 0.000000 MiB config = config or {}
74
75 102.769531 MiB 0.000000 MiB with rasterio.Env(**config):
76 102.957031 MiB 0.187500 MiB with rasterio.open(src_path) as src_dst:
77 102.957031 MiB 0.000000 MiB meta = src_dst.meta
78 102.957031 MiB 0.000000 MiB indexes = indexes if indexes else src_dst.indexes
79 102.957031 MiB 0.000000 MiB nodata = nodata if nodata is not None else src_dst.nodata
80 103.000000 MiB 0.042969 MiB alpha = has_alpha_band(src_dst)
81 103.000000 MiB 0.000000 MiB mask = has_mask_band(src_dst)
82
83 103.000000 MiB 0.000000 MiB if not add_mask and (
84 103.000000 MiB 0.000000 MiB (nodata is not None or alpha)
85 103.000000 MiB 0.000000 MiB and dst_kwargs.get("compress") in ["JPEG", "jpeg"]
86 ):
87 warnings.warn(
88 "Using lossy compression with Nodata or Alpha band "
89 "can results in unwanted artefacts.",
90 LossyCompression,
91 )
92
93 103.000000 MiB 0.000000 MiB if overview_level is None:
94 103.000000 MiB 0.000000 MiB overview_level = get_maximum_overview_level(
95 103.000000 MiB 0.000000 MiB src_dst,
96 103.000000 MiB 0.000000 MiB min(int(dst_kwargs["blockxsize"]), int(dst_kwargs["blockysize"])),
97 )
98
99 103.000000 MiB 0.000000 MiB vrt_params = dict(add_alpha=True)
100
101 103.000000 MiB 0.000000 MiB if nodata is not None:
102 103.000000 MiB 0.000000 MiB vrt_params.update(
103 103.000000 MiB 0.000000 MiB dict(nodata=nodata, add_alpha=False, src_nodata=nodata)
104 )
105
106 103.000000 MiB 0.000000 MiB if alpha:
107 vrt_params.update(dict(add_alpha=False))
108
109 103.308594 MiB 0.308594 MiB with WarpedVRT(src_dst, **vrt_params) as vrt_dst:
110 103.308594 MiB 0.000000 MiB meta = vrt_dst.meta
111 103.308594 MiB 0.000000 MiB meta["count"] = len(indexes)
112
113 103.308594 MiB 0.000000 MiB if add_mask:
114 meta.pop("nodata", None)
115 meta.pop("alpha", None)
116
117 103.308594 MiB 0.000000 MiB meta.update(**dst_kwargs)
118 103.308594 MiB 0.000000 MiB meta.pop("compress", None)
119 103.308594 MiB 0.000000 MiB meta.pop("photometric", None)
120
121 103.308594 MiB 0.000000 MiB if in_memory is None:
122 in_memory = vrt_dst.width * vrt_dst.height < IN_MEMORY_THRESHOLD
123
124 103.308594 MiB 0.000000 MiB with ExitStack() as ctx:
125 103.308594 MiB 0.000000 MiB if in_memory:
126 103.320312 MiB 0.011719 MiB tmpfile = ctx.enter_context(MemoryFile())
127 103.914062 MiB 0.593750 MiB tmp_dst = ctx.enter_context(tmpfile.open(**meta))
128 else:
129 tmpfile = ctx.enter_context(tempfile.NamedTemporaryFile())
130 tmp_dst = ctx.enter_context(rasterio.open(tmpfile, "w", **meta))
131
132 103.937500 MiB 0.023438 MiB wind = list(tmp_dst.block_windows(1))
133
134 103.937500 MiB 0.000000 MiB if not quiet:
135 103.945312 MiB 0.007812 MiB click.echo("Reading input: {}".format(src_path), err=True)
136 103.945312 MiB 0.000000 MiB fout = os.devnull if quiet else sys.stderr
137 103.945312 MiB 0.000000 MiB with click.progressbar(
138 103.992188 MiB 0.046875 MiB wind, length=len(wind), file=fout, show_percent=True
139 103.992188 MiB 0.000000 MiB ) as windows:
140 559.808594 MiB 0.003906 MiB for ij, w in windows:
141 559.304688 MiB 1.761719 MiB matrix = vrt_dst.read(window=w, indexes=indexes)
142 559.804688 MiB 0.527344 MiB tmp_dst.write(matrix, window=w)
143
144 559.804688 MiB 0.000000 MiB if add_mask or mask:
145 mask_value = vrt_dst.dataset_mask(window=w)
146 tmp_dst.write_mask(mask_value, window=w)
147
148 559.808594 MiB 0.000000 MiB if not quiet:
149 559.812500 MiB 0.003906 MiB click.echo("Adding overviews...", err=True)
150 559.816406 MiB 0.003906 MiB overviews = [2 ** j for j in range(1, overview_level + 1)]
151 788.105469 MiB 228.289062 MiB tmp_dst.build_overviews(overviews, Resampling[overview_resampling])
152
153 788.105469 MiB 0.000000 MiB if not quiet:
154 788.105469 MiB 0.000000 MiB click.echo("Updating dataset tags...", err=True)
155
156 788.113281 MiB 0.000000 MiB for i, b in enumerate(indexes):
157 788.113281 MiB 0.007812 MiB tmp_dst.set_band_description(i + 1, src_dst.descriptions[b - 1])
158
159 788.117188 MiB 0.003906 MiB tags = src_dst.tags()
160 788.117188 MiB 0.000000 MiB tags.update(
161 788.117188 MiB 0.000000 MiB dict(
162 788.117188 MiB 0.000000 MiB OVR_RESAMPLING_ALG=Resampling[
163 788.117188 MiB 0.000000 MiB overview_resampling
164 ].name.upper()
165 )
166 )
167 788.125000 MiB 0.007812 MiB tmp_dst.update_tags(**tags)
168
169 788.125000 MiB 0.000000 MiB if not quiet:
170 788.125000 MiB 0.000000 MiB click.echo("Writing output to: {}".format(dst_path), err=True)
171 150.285156 MiB 0.000000 MiB copy(tmp_dst, dst_path, copy_src_overviews=True, **dst_kwargs)
Raw
in-temp.txt
# TMP
Line # Mem usage Increment Line Contents
================================================
29 102.656250 MiB 102.656250 MiB @profile(precision=6)
30 def cog_translate(
31 src_path,
32 dst_path,
33 dst_kwargs,
34 indexes=None,
35 nodata=None,
36 add_mask=None,
37 overview_level=None,
38 overview_resampling="nearest",
39 in_memory=None,
40 config=None,
41 quiet=False,
42 ):
43 """
44 Create Cloud Optimized Geotiff.
45
46 Parameters
47 ----------
48 src_path : str or PathLike object
49 A dataset path or URL. Will be opened in "r" mode.
50 dst_path : str or Path-like object
51 An output dataset path or or PathLike object.
52 Will be opened in "w" mode.
53 dst_kwargs: dict
54 output dataset creation options.
55 indexes : tuple, int, optional
56 Raster band indexes to copy.
57 nodata, int, optional
58 Overwrite nodata masking values for input dataset.
59 add_mask, bool, optional
60 Force output dataset creation with a mask.
61 overview_level : int, optional (default: 6)
62 COGEO overview (decimation) level
63 overview_resampling : str, optional (default: "nearest")
64 Resampling algorithm for overviews
65 in_memory: bool, optional
66 Force processing raster in memory (default: process in memory if smal)
67 config : dict
68 Rasterio Env options.
69 quiet: bool, optional (default: False)
70 Mask processing steps.
71
72 """
73 102.656250 MiB 0.000000 MiB config = config or {}
74
75 102.656250 MiB 0.000000 MiB with rasterio.Env(**config):
76 102.835938 MiB 0.179688 MiB with rasterio.open(src_path) as src_dst:
77 102.835938 MiB 0.000000 MiB meta = src_dst.meta
78 102.835938 MiB 0.000000 MiB indexes = indexes if indexes else src_dst.indexes
79 102.835938 MiB 0.000000 MiB nodata = nodata if nodata is not None else src_dst.nodata
80 102.878906 MiB 0.042969 MiB alpha = has_alpha_band(src_dst)
81 102.878906 MiB 0.000000 MiB mask = has_mask_band(src_dst)
82
83 102.878906 MiB 0.000000 MiB if not add_mask and (
84 102.878906 MiB 0.000000 MiB (nodata is not None or alpha)
85 102.882812 MiB 0.003906 MiB and dst_kwargs.get("compress") in ["JPEG", "jpeg"]
86 ):
87 warnings.warn(
88 "Using lossy compression with Nodata or Alpha band "
89 "can results in unwanted artefacts.",
90 LossyCompression,
91 )
92
93 102.882812 MiB 0.000000 MiB if overview_level is None:
94 102.882812 MiB 0.000000 MiB overview_level = get_maximum_overview_level(
95 102.882812 MiB 0.000000 MiB src_dst,
96 102.882812 MiB 0.000000 MiB min(int(dst_kwargs["blockxsize"]), int(dst_kwargs["blockysize"])),
97 )
98
99 102.882812 MiB 0.000000 MiB vrt_params = dict(add_alpha=True)
100
101 102.882812 MiB 0.000000 MiB if nodata is not None:
102 102.882812 MiB 0.000000 MiB vrt_params.update(
103 102.882812 MiB 0.000000 MiB dict(nodata=nodata, add_alpha=False, src_nodata=nodata)
104 )
105
106 102.882812 MiB 0.000000 MiB if alpha:
107 vrt_params.update(dict(add_alpha=False))
108
109 103.218750 MiB 0.335938 MiB with WarpedVRT(src_dst, **vrt_params) as vrt_dst:
110 103.218750 MiB 0.000000 MiB meta = vrt_dst.meta
111 103.218750 MiB 0.000000 MiB meta["count"] = len(indexes)
112
113 103.218750 MiB 0.000000 MiB if add_mask:
114 meta.pop("nodata", None)
115 meta.pop("alpha", None)
116
117 103.218750 MiB 0.000000 MiB meta.update(**dst_kwargs)
118 103.218750 MiB 0.000000 MiB meta.pop("compress", None)
119 103.218750 MiB 0.000000 MiB meta.pop("photometric", None)
120
121 103.218750 MiB 0.000000 MiB if in_memory is None:
122 in_memory = vrt_dst.width * vrt_dst.height < IN_MEMORY_THRESHOLD
123
124 103.218750 MiB 0.000000 MiB with ExitStack() as ctx:
125 103.218750 MiB 0.000000 MiB if in_memory:
126 tmpfile = ctx.enter_context(MemoryFile())
127 tmp_dst = ctx.enter_context(tmpfile.open(**meta))
128 else:
129 103.222656 MiB 0.003906 MiB tmpfile = ctx.enter_context(tempfile.NamedTemporaryFile())
130 103.843750 MiB 0.621094 MiB tmp_dst = ctx.enter_context(rasterio.open(tmpfile, "w", **meta))
131
132 103.863281 MiB 0.019531 MiB wind = list(tmp_dst.block_windows(1))
133
134 103.863281 MiB 0.000000 MiB if not quiet:
135 103.867188 MiB 0.003906 MiB click.echo("Reading input: {}".format(src_path), err=True)
136 103.867188 MiB 0.000000 MiB fout = os.devnull if quiet else sys.stderr
137 103.867188 MiB 0.000000 MiB with click.progressbar(
138 103.898438 MiB 0.031250 MiB wind, length=len(wind), file=fout, show_percent=True
139 103.898438 MiB 0.000000 MiB ) as windows:
140 557.535156 MiB 0.003906 MiB for ij, w in windows:
141 557.031250 MiB 1.757812 MiB matrix = vrt_dst.read(window=w, indexes=indexes)
142 557.531250 MiB 0.523438 MiB tmp_dst.write(matrix, window=w)
143
144 557.531250 MiB 0.000000 MiB if add_mask or mask:
145 mask_value = vrt_dst.dataset_mask(window=w)
146 tmp_dst.write_mask(mask_value, window=w)
147
148 557.535156 MiB 0.000000 MiB if not quiet:
149 557.535156 MiB 0.000000 MiB click.echo("Adding overviews...", err=True)
150 557.539062 MiB 0.003906 MiB overviews = [2 ** j for j in range(1, overview_level + 1)]
151 571.160156 MiB 13.621094 MiB tmp_dst.build_overviews(overviews, Resampling[overview_resampling])
152
153 571.160156 MiB 0.000000 MiB if not quiet:
154 571.160156 MiB 0.000000 MiB click.echo("Updating dataset tags...", err=True)
155
156 571.167969 MiB 0.000000 MiB for i, b in enumerate(indexes):
157 571.167969 MiB 0.007812 MiB tmp_dst.set_band_description(i + 1, src_dst.descriptions[b - 1])
158
159 571.171875 MiB 0.003906 MiB tags = src_dst.tags()
160 571.171875 MiB 0.000000 MiB tags.update(
161 571.171875 MiB 0.000000 MiB dict(
162 571.171875 MiB 0.000000 MiB OVR_RESAMPLING_ALG=Resampling[
163 571.171875 MiB 0.000000 MiB overview_resampling
164 ].name.upper()
165 )
166 )
167 571.179688 MiB 0.007812 MiB tmp_dst.update_tags(**tags)
168
169 571.179688 MiB 0.000000 MiB if not quiet:
170 571.179688 MiB 0.000000 MiB click.echo("Writing output to: {}".format(dst_path), err=True)
171 115.421875 MiB 0.000000 MiB copy(tmp_dst, dst_path, copy_src_overviews=True, **dst_kwargs)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment