Merge pull request #202 from sat-utils/develop

Release 0.3.0

Author: matthewhanson

.circleci/config.yml

@@ -52,7 +52,7 @@ jobs:
 
   docker_build_and_test:
     machine:
-      docker_layer_caching: true
+      docker_layer_caching: false
     steps:
       - *restore_repo
       - checkout
@@ -68,7 +68,7 @@ jobs:
 
   docker_deploy_production:
     machine:
-      docker_layer_caching: true
+      docker_layer_caching: false
     steps:
       - *restore_repo
       - run:
@@ -86,7 +86,7 @@ jobs:
 
   docker_deploy_develop:
     machine:
-      docker_layer_caching: true
+      docker_layer_caching: false
     steps:
       - *restore_repo
       - run:

CHANGELOG.md

@@ -4,10 +4,36 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+## [v0.3.0] - 2019-10-16
+
+### Added
+- /stac/search linked to from /stac
+- New `ids` parameter added for searching by IDs
+- New `collections` parameter added for searching list of specific collections
+- SATAPI_COLLECTION_LIMIT environment variable added for the number of collections to return at the /stac and /collections endpoints. Since pagination is not supported at these endpoints this should be set higher than the number of collections available. Defaults to 100.
+
+### Changed
+- Fields parameter changed to search on any fields rather than just fields under `properties`. Field under `properties` must now be referenced by `properties.<fieldname>`
+- `collection` now a top level field rather than a property
+
+
 ## [v0.2.6] - 2019-08-28
 
 ### Added
 - Support for `in` operator on property fields
+- SATAPI_COLLECTION_LIMIT environment variable added for the number of collections to return at the /stac and /collections endpoints. Since pagination is not supported at these endpoints this should be set higher than the number of collections available. Defaults to 100.
+
+### Added
+- /stac/search linked to from /stac
+- New `ids` parameter added for searching by IDs
+- New `collections` parameter added for searching list of specific collections
+
+### Changed
+- Fields parameter changed to search on any fields rather than just fields under `properties`. `property` fields must now be referenced by `property.fieldname`
+- `collection` now a top level field rather than a property
+
 
 ## [v0.2.5] - 2019-06-21
 
@@ -108,6 +134,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Refactor and improve splitting
 
 [Unreleased]: https://github.com/sat-utils/sat-api/compare/master...develop
+[v0.3.0]: https://github.com/sat-utils/sat-api/compare/v0.2.6...v0.3.0
+[v0.2.6]: https://github.com/sat-utils/sat-api/compare/v0.2.5...v0.2.6
 [v0.2.5]: https://github.com/sat-utils/sat-api/compare/v0.2.4...v0.2.5
 [v0.2.4]: https://github.com/sat-utils/sat-api/compare/v0.2.3...v0.2.4
 [v0.2.3]: https://github.com/sat-utils/sat-api/compare/v0.2.2...v0.2.3

README.md

@@ -12,6 +12,26 @@ The STAC version supported by a given version of sat-api is shown in the table b
 | -------- | ----  |
 | 0.1.0    | 0.5.x |
 | 0.2.x    | 0.6.x |
+| 0.3.x    | 0.7.x |
+
+## Usage
+
+This repository contains just the Node libraries for running the API. The [sat-api-deployment](https://github.com/sat-utils/sat-api-deployment) repository is for deploying sat-api to AWS.
+
+### Environment variables
+
+There are some environment variables used in the code. Some do not have defaults and must be set.
+
+| Name | Description | Default Value |
+| ---- | ----------- | ------------- |
+| STAC_ID | ID of this catalog | - |
+| STAC_TITLE | Title of this catalog | - |
+| STAC_DESCRIPTION | Description of this catalog | - |
+| STAC_DOCS_URL | URL to documentation | - |
+| SATAPI_URL | The root endpoint of this API to use for links | Inferred from request |
+| ES_BATCH_SIZE | Number of records to ingest in single batch | 500 |
+| SATAPI_ES_PRECISION | Precision to use for geometry queries, see [ES documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/geo-shape.html) | '5mi' |
+| LOG_LEVEL | Level for logging | 'info' |
 
 
 ## Development

lerna.json

@@ -1,6 +1,6 @@
 {
   "lerna": "2.11.0",
-  "version": "0.2.6",
+  "version": "0.3.0",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/api-lib/libs/api.js

@@ -92,6 +92,19 @@ const extractFields = function (params) {
   return fieldRules
 }
 
+const extractIds = function (params) {
+  let idsRules
+  const { ids } = params
+  if (ids) {
+    if (typeof ids === 'string') {
+      idsRules = JSON.parse(ids)
+    } else {
+      idsRules = ids.slice()
+    }
+  }
+  return idsRules
+}
+
 const parsePath = function (path) {
   const searchFilters = {
     stac: false,
@@ -150,8 +163,10 @@ const addCollectionLinks = function (results, endpoint) {
 // Impure - mutates results
 const addItemLinks = function (results, endpoint) {
   results.forEach((result) => {
-    const { id, links } = result
-    const { collection } = result.properties
+    let { links } = result
+    const { id, collection } = result
+
+    links = (links === undefined) ? [] : links
     // self link
     links.splice(0, 0, {
       rel: 'self',
@@ -220,6 +235,10 @@ const collectionsToCatalogLinks = function (results, endpoint) {
     rel: 'self',
     href: `${endpoint}/stac`
   })
+  catalog.links.push({
+    rel: 'search',
+    href: `${endpoint}/stac/search`
+  })
   return catalog
 }
 
@@ -302,14 +321,17 @@ const search = async function (
     const intersects = hasIntersects || bbox
     const query = extractStacQuery(queryParameters)
     const fields = extractFields(queryParameters)
+    const ids = extractIds(queryParameters)
     const parameters = {
       datetime,
       intersects,
       query,
       sort,
-      fields
+      fields,
+      ids
     }
-    // Keep only exisiting parameters
+    const colLimit = process.env.SATAPI_COLLECTION_LIMIT || 100
+    // Keep only existing parameters
     const searchParameters = Object.keys(parameters)
       .filter((key) => parameters[key])
       .reduce((obj, key) => ({
@@ -323,7 +345,7 @@ const search = async function (
     // Root catalog with collection links
     if (stac && !searchPath) {
       const { results } =
-        await backend.search({}, 'collections', page, limit)
+        await backend.search({}, 'collections', page, colLimit)
       apiResponse = collectionsToCatalogLinks(results, endpoint)
     }
     // STAC Search
@@ -335,7 +357,7 @@ const search = async function (
     // All collections
     if (collections && !collectionId) {
       const { results, meta } =
-        await backend.search({}, 'collections', page, limit)
+        await backend.search({}, 'collections', page, colLimit)
       const linkedCollections = addCollectionLinks(results, endpoint)
       apiResponse = { meta, collections: linkedCollections }
     }
@@ -355,9 +377,9 @@ const search = async function (
     // Items in a collection
     if (collections && collectionId && items && !itemId) {
       const updatedQuery = Object.assign({}, searchParameters.query, {
-        collection: {
-          eq: collectionId
-        }
+        collections: [
+          collectionId
+        ]
       })
       const itemIdParameters = Object.assign(
         {}, searchParameters, { query: updatedQuery }

packages/api-lib/libs/es.js

@@ -79,7 +79,6 @@ async function prepare(index) {
   const props = {
     'type': 'object',
     properties: {
-      'collection': { type: 'keyword' },
       'datetime': { type: 'date' },
       'eo:cloud_cover': { type: 'float' },
       'eo:gsd': { type: 'float' },
@@ -117,6 +116,7 @@ async function prepare(index) {
             dynamic_templates: dynamicTemplates,
             properties: {
               'id': { type: 'keyword' },
+              'collection': { type: 'keyword' },
               'properties': props,
               geometry: {
                 type: 'geo_shape',
@@ -170,7 +170,7 @@ async function _stream() {
       const links = data.links.filter((link) => hlinks.includes(link))
       let esDataObject = Object.assign({}, data, { links })
       if (index === 'items') {
-        const collectionId = data.properties.collection
+        const collectionId = data.collection
         const itemCollection =
           collections.find((collection) => (collectionId === collection.id))
         if (itemCollection) {
@@ -266,7 +266,9 @@ function buildQuery(parameters) {
   const inop = 'in'
   const { query, intersects } = parameters
   let must = []
+  const should = []
   if (query) {
+    const { collections } = query
     // Using reduce rather than map as we don't currently support all
     // stac query operators.
     must = Object.keys(query).reduce((accumulator, property) => {
@@ -294,8 +296,15 @@ function buildQuery(parameters) {
       }
       return accumulator
     }, must)
+
+    if (collections) {
+      collections.forEach((collection) => {
+        should.push({ term: { 'collection': collection } })
+      })
+    }
   }
 
+
   if (intersects) {
     const { geometry } = intersects
     must.push({
@@ -310,7 +319,7 @@ function buildQuery(parameters) {
     must.push(datetimeQuery)
   }
 
-  const filter = { bool: { must } }
+  const filter = { bool: { must, should } }
   const queryBody = {
     constant_score: { filter }
   }
@@ -331,6 +340,16 @@ function buildIdQuery(id) {
   }
 }
 
+function buildIdsQuery(ids) {
+  return {
+    query: {
+      ids: {
+        values: ids
+      }
+    }
+  }
+}
+
 function buildSort(parameters) {
   const { sort } = parameters
   let sorting
@@ -355,46 +374,43 @@ function buildSort(parameters) {
 
 function buildFieldsFilter(parameters) {
   const id = 'id'
-  const type = 'type'
-  const bbox = 'bbox'
-  const links = 'links'
-  const assets = 'assets'
   const { fields } = parameters
   const _sourceInclude = []
   const _sourceExclude = []
   if (fields) {
-    const { geometry, includes, excludes } = fields
-    if (typeof geometry !== 'undefined' && !geometry) {
-      _sourceExclude.push('geometry')
-    }
-    if (includes && includes.length > 0) {
-      const propertiesIncludes = includes.map(
-        (field) => (`properties.${field}`)
+    const { include, exclude } = fields
+    if (include && include.length > 0) {
+      const propertiesIncludes = include.map(
+        (field) => (`${field}`)
       ).concat(
-        [id, type, bbox, links, assets]
+        [id]
       )
       _sourceInclude.push(...propertiesIncludes)
     }
-    if (excludes && excludes.length > 0) {
-      const filteredExcludes = excludes.filter((field) =>
-        (![id, type, bbox, links, assets].includes(field)))
-      const propertiesExcludes = filteredExcludes.map((field) => (`properties.${field}`))
-      _sourceExclude.push(...propertiesExcludes)
+    if (exclude && exclude.length > 0) {
+      const filteredExcludes = exclude.filter((field) =>
+        (![id].includes(field)))
+      const propertiesExclude = filteredExcludes.map((field) => (`${field}`))
+      _sourceExclude.push(...propertiesExclude)
     }
   }
   return { _sourceExclude, _sourceInclude }
 }
 
 async function search(parameters, index = '*', page = 1, limit = 10) {
   let body
-  if (parameters.id) {
+  if (parameters.ids) {
+    const { ids } = parameters
+    body = buildIdsQuery(ids)
+  } else if (parameters.id) {
     const { id } = parameters
     body = buildIdQuery(id)
   } else {
     body = buildQuery(parameters)
   }
   const sort = buildSort(parameters)
   body.sort = sort
+  logger.info(`Elasticsearch query: ${JSON.stringify(body)}`)
 
   const searchParams = {
     index,

packages/api-lib/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sat-utils/api-lib",
-  "version": "0.2.6",
+  "version": "0.3.0",
   "description": "A library for creating a search API of public Satellites metadata using Elasticsearch",
   "main": "index.js",
   "scripts": {

packages/api-lib/tests/fixtures/item.json

@@ -149,8 +149,8 @@
     },
     "id": "LC80100102015082LGN00",
     "links": [],
+    "collection": "landsat-8-l1",
     "properties": {
-        "collection": "landsat-8-l1",
         "datetime": "2015-03-23T15:05:56.200728+00:00",
         "eo:cloud_cover": 8.26,
         "eo:epsg": 32622,

packages/api-lib/tests/fixtures/stac/LC80100102015050LGN00.json

@@ -166,8 +166,8 @@
             "rel": "collection"
         }
     ],
+    "collection": "landsat-8-l1",
     "properties": {
-        "collection": "landsat-8-l1",
         "datetime": "2015-02-19T15:06:12.565047+00:00",
         "eo:cloud_cover": 0.54,
         "eo:epsg": 32622,

packages/api-lib/tests/fixtures/stac/LC80100102015082LGN00.json

@@ -166,8 +166,8 @@
             "rel": "collection"
         }
     ],
+    "collection": "landsat-8-l1",
     "properties": {
-        "collection": "landsat-8-l1",
         "datetime": "2015-03-23T15:05:56.200728+00:00",
         "eo:cloud_cover": 8.26,
         "eo:epsg": 32622,