Skip to content

Commit 8f2442c

Browse files
matthewhansonmatthewhanson
authored
Merge pull request #138 from sat-utils/develop
publish 0.2.1
2 parents 6a441fe + 53d7eeb commit 8f2442c

File tree

11 files changed

+343
-20
lines changed

11 files changed

+343
-20
lines changed

‎CHANGELOG.md

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
66

77
## [Unreleased]
88

9+
## [v0.2.1] - 2019-01-17
910

10-
## [v0.2.0] - 2018-11-02
11+
### Fixed
12+
- Error handling of Items failing to get written to Elasticsearch (such as when unable to parse geometry). Now will continue traversing catalog.
13+
14+
15+
## [v0.2.0] - 2019-01-16
1116

1217
### Changed
1318
- Implemented the changes in [STAC 0.6.0](https://github.com/radiantearth/stac-spec/blob/master/CHANGELOG.md)
@@ -64,6 +69,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
6469
- Refactor and improve splitting
6570

6671
[Unreleased]: https://github.com/sat-utils/sat-api/compare/master...develop
72+
[v0.2.1]: https://github.com/sat-utils/sat-api/compare/v0.2.0...v0.2.1
6773
[v0.2.0]: https://github.com/sat-utils/sat-api/compare/v0.1.0...v0.2.0
6874
[v0.1.0]: https://github.com/sat-utils/sat-api/compare/v0.0.2...v0.1.0
6975
[v0.0.2]: https://github.com/sat-utils/sat-api/compare/legacy-v2.0.0...v0.0.2

‎lerna.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"lerna": "2.11.0",
3-
"version": "0.2.0",
3+
"version": "0.2.1",
44
"npmClient": "yarn",
55
"packages": [
66
"packages/*"

‎packages/api-lib/libs/ElasticSearchWriteableStream.js

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,13 @@ class ElasticSearchWritableStream extends stream.Writable {
5656
async _writev(records, next) {
5757
const body = this.transformRecords(records)
5858
try {
59-
await this.client.bulk({ body })
60-
logger.debug(`Wrote batch of documents size ${body.length / 2}`)
59+
const result = await this.client.bulk({ body })
60+
const { errors, items } = result
61+
if (errors) {
62+
logger.error(items)
63+
} else {
64+
logger.debug(`Wrote batch of documents size ${body.length / 2}`)
65+
}
6166
next()
6267
} catch (err) {
6368
next(err)

‎packages/api-lib/libs/ingest.js

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -92,16 +92,10 @@ async function visit(url, stream, recursive, collectionsOnly) {
9292
while (stack.length) {
9393
const node = stack.pop()
9494
const isCollection = node.hasOwnProperty('extent')
95-
const written = stream.write(node)
95+
stream.write(node)
9696
if (recursive && !(isCollection && collectionsOnly)) {
97-
if (written) {
98-
// eslint-disable-next-line
99-
await visitChildren(node, stack, visited, basePath)
100-
} else {
101-
stream.once('drain', async () => {
102-
await visitChildren(node, stack, visited, basePath)
103-
})
104-
}
97+
// eslint-disable-next-line
98+
await visitChildren(node, stack, visited, basePath)
10599
}
106100
}
107101
stream.push(null)

‎packages/api-lib/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@sat-utils/api-lib",
3-
"version": "0.2.0",
3+
"version": "0.2.1",
44
"description": "A library for creating a search API of public Satellites metadata using Elasticsearch",
55
"main": "index.js",
66
"scripts": {

‎packages/api-lib/tests/fixtures/stac/badGeometryItem.json

Lines changed: 314 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,314 @@
1+
{
2+
"assets": {
3+
"ANG": {
4+
"href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_ANG.txt",
5+
"title": "Angle coefficients file",
6+
"type": "text/plain"
7+
},
8+
"B1": {
9+
"eo:bands": [
10+
0
11+
],
12+
"href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_B1.TIF",
13+
"title": "Band 1 (coastal)",
14+
"type": "image/x.geotiff"
15+
},
16+
"B10": {
17+
"eo:bands": [
18+
9
19+
],
20+
"href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_B10.TIF",
21+
"title": "Band 10 (lwir)",
22+
"type": "image/x.geotiff"
23+
},
24+
"B11": {
25+
"eo:bands": [
26+
10
27+
],
28+
"href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_B11.TIF",
29+
"title": "Band 11 (lwir)",
30+
"type": "image/x.geotiff"
31+
},
32+
"B2": {
33+
"eo:bands": [
34+
1
35+
],
36+
"href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_B2.TIF",
37+
"title": "Band 2 (blue)",
38+
"type": "image/x.geotiff"
39+
},
40+
"B3": {
41+
"eo:bands": [
42+
2
43+
],
44+
"href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_B3.TIF",
45+
"title": "Band 3 (green)",
46+
"type": "image/x.geotiff"
47+
},
48+
"B4": {
49+
"eo:bands": [
50+
3
51+
],
52+
"href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_B4.TIF",
53+
"title": "Band 4 (red)",
54+
"type": "image/x.geotiff"
55+
},
56+
"B5": {
57+
"eo:bands": [
58+
4
59+
],
60+
"href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_B5.TIF",
61+
"title": "Band 5 (nir)",
62+
"type": "image/x.geotiff"
63+
},
64+
"B6": {
65+
"eo:bands": [
66+
5
67+
],
68+
"href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_B6.TIF",
69+
"title": "Band 6 (swir16)",
70+
"type": "image/x.geotiff"
71+
},
72+
"B7": {
73+
"eo:bands": [
74+
6
75+
],
76+
"href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_B7.TIF",
77+
"title": "Band 7 (swir22)",
78+
"type": "image/x.geotiff"
79+
},
80+
"B8": {
81+
"eo:bands": [
82+
7
83+
],
84+
"href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_B8.TIF",
85+
"title": "Band 8 (pan)",
86+
"type": "image/x.geotiff"
87+
},
88+
"B9": {
89+
"eo:bands": [
90+
8
91+
],
92+
"href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_B9.TIF",
93+
"title": "Band 9 (cirrus)",
94+
"type": "image/x.geotiff"
95+
},
96+
"BQA": {
97+
"href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_BQA.TIF",
98+
"title": "Band quality data",
99+
"type": "image/x.geotiff"
100+
},
101+
"MTL": {
102+
"href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_MTL.txt",
103+
"title": "original metadata file",
104+
"type": "text/plain"
105+
},
106+
"index": {
107+
"href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/index.html",
108+
"title": "HTML index page",
109+
"type": "text/html"
110+
},
111+
"thumbnail": {
112+
"href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_thumb_large.jpg",
113+
"title": "Thumbnail image",
114+
"type": "image/jpeg"
115+
}
116+
},
117+
"bbox": [
118+
-173.27809,
119+
72.22415,
120+
178.89739,
121+
74.62735
122+
],
123+
"geometry": {
124+
"coordinates": [
125+
[
126+
[
127+
[
128+
-180.0,
129+
73.00696964504284
130+
],
131+
[
132+
-180.0,
133+
73.09357289928789
134+
],
135+
[
136+
-180.0,
137+
73.4022696917215
138+
],
139+
[
140+
-180.0,
141+
73.70257483495145
142+
],
143+
[
144+
-180.0,
145+
73.7893042253521
146+
],
147+
[
148+
-179.99650733482093,
149+
73.79109483116221
150+
],
151+
[
152+
-178.66433925079528,
153+
74.47406551085987
154+
],
155+
[
156+
-178.60228746524575,
157+
74.50587797556413
158+
],
159+
[
160+
-178.533,
161+
74.5414
162+
],
163+
[
164+
-178.52333378926406,
165+
74.53979633032269
166+
],
167+
[
168+
-177.0035,
169+
74.2876487795428
170+
],
171+
[
172+
-176.7359660507269,
173+
74.24326364313104
174+
],
175+
[
176+
-174.01065330156592,
177+
73.79112148149187
178+
],
179+
[
180+
-173.371,
181+
73.685
182+
],
183+
[
184+
-173.45037750938653,
185+
73.64971591347567
186+
],
187+
[
188+
-173.60275118996861,
189+
73.58198430824847
190+
],
191+
[
192+
-175.0997578783975,
193+
72.9165500570324
194+
],
195+
[
196+
-175.40543925453161,
197+
72.78067166770337
198+
],
199+
[
200+
-176.18673897068663,
201+
72.43337623100723
202+
],
203+
[
204+
-176.258,
205+
72.4017
206+
],
207+
[
208+
-177.00319088157397,
209+
72.52223485312118
210+
],
211+
[
212+
-177.26950829931573,
213+
72.56531177693338
214+
],
215+
[
216+
-178.60216354768573,
217+
72.78086917114648
218+
],
219+
[
220+
-180.0,
221+
73.00696964504284
222+
]
223+
]
224+
],
225+
[
226+
[
227+
[
228+
180.0,
229+
73.7893042253521
230+
],
231+
[
232+
180.0,
233+
73.70257483495145
234+
],
235+
[
236+
180.0,
237+
73.40226969172153
238+
],
239+
[
240+
180.0,
241+
73.09357289928789
242+
],
243+
[
244+
180.0,
245+
73.00696964504284
246+
],
247+
[
248+
178.84742274390246,
249+
73.19339936890243
250+
],
251+
[
252+
178.84,
253+
73.1946
254+
],
255+
[
256+
178.90331756836056,
257+
73.227061401244
258+
],
259+
[
260+
178.96865829339555,
261+
73.2605600264732
262+
],
263+
[
264+
179.59566838898684,
265+
73.58201308956507
266+
],
267+
[
268+
179.75007781076354,
269+
73.66117510298298
270+
],
271+
[
272+
180.0,
273+
73.7893042253521
274+
]
275+
]
276+
]
277+
],
278+
"type": "Polygon"
279+
},
280+
"id": "badGeometryItem",
281+
"links": [
282+
{
283+
"href": "badGeometryItem.json",
284+
"rel": "self"
285+
},
286+
{
287+
"href": "catalog.json",
288+
"rel": "root"
289+
},
290+
{
291+
"href": "collection.json",
292+
"rel": "parent"
293+
},
294+
{
295+
"href": "collection.json",
296+
"rel": "collection"
297+
}
298+
],
299+
"properties": {
300+
"collection": "landsat-8-l1",
301+
"datetime": "2014-07-04T23:56:54.593296+00:00",
302+
"eo:cloud_cover": 65,
303+
"eo:column": "096",
304+
"eo:epsg": 3261,
305+
"eo:row": "008",
306+
"eo:sun_azimuth": -178.92904262,
307+
"eo:sun_elevation": 39.43839572,
308+
"landsat:processing_level": "L1GT",
309+
"landsat:product_id": null,
310+
"landsat:scene_id": "LC80960082014185LGN00",
311+
"landsat:tier": "pre-collection"
312+
},
313+
"type": "Feature"
314+
}

0 commit comments

Comments
 (0)