Last active
November 23, 2021 08:02
-
-
Save thomasneirynck/4d4bc0e0a7bd0f154aa31fa556c17c29 to your computer and use it in GitHub Desktop.
Ingest geojson file into Elasticsearch index. Each feature in the FeatureCollection corresponds to a document in Elasticsearch.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| const fs = require("fs"); | |
| const elasticsearch = require('elasticsearch'); | |
| const oboe = require('oboe'); | |
| const geojsonInput = process.argv[2] || 'feature_collection.geojson'; | |
| const indexname = process.argv[3] || geojsonInput.split('.')[0] || 'feature_collection'; | |
| const geometryFieldName = 'geometry'; | |
| const shape_type = process.argv[4] || 'geo_shape'; | |
| if (shape_type !== 'geo_point' && shape_type !== 'geo_shape') { | |
| console.error(`Invalid shapetype ${shape_type}`); | |
| return; | |
| } | |
| async function ingest() { | |
| const esClient = new elasticsearch.Client({ | |
| host: 'elastic:changeme@localhost:9200', | |
| version: '7.1', | |
| // log: 'trace' | |
| }); | |
| try { | |
| await esClient.ping({ | |
| requestTimeout: 1000 | |
| }); | |
| } catch (e) { | |
| console.error('Cannot reach Elasticsearch', e); | |
| throw e; | |
| } | |
| try { | |
| await esClient.indices.delete({ | |
| index: indexname | |
| }); | |
| } catch (e) { | |
| console.warn(e); | |
| } | |
| try { | |
| console.log('shape type', shape_type); | |
| await esClient.indices.create({ | |
| index: indexname, | |
| body: { | |
| mappings: { | |
| "properties": { | |
| [geometryFieldName]: { | |
| "type": shape_type, | |
| "ignore_malformed": true | |
| } | |
| } | |
| } | |
| } | |
| }); | |
| } catch (e) { | |
| console.error(e); | |
| throw e; | |
| } | |
| const readStream = fs.createReadStream(geojsonInput); | |
| let i = 0; | |
| oboe(readStream) | |
| .node('features.*',async (feature) => { | |
| const geometry = shape_type === 'geo_point' ? feature.geometry.coordinates : feature.geometry; | |
| const doc = { | |
| ...feature.properties, | |
| [geometryFieldName]: geometry | |
| }; | |
| try { | |
| await esClient.create({ | |
| id: i++, | |
| index: indexname, | |
| // type: '_doc', //ES 80 does not use types | |
| body: doc | |
| }); | |
| console.log(`Created ${i}`); | |
| } catch (e) { | |
| console.error(e); | |
| throw e; | |
| } | |
| }) | |
| .done(()=>{ | |
| console.log('done processing'); | |
| }); | |
| } | |
| ingest(); |
question; do you have anything like this written in python?
@Konecny2k no I don't. But there is a python client for Elasticsearch https://elasticsearch-py.readthedocs.io/en/v7.12.0/ . That will allow you to build nearly the same thing.
You could also use GDAL to ingest geo-data https://www.elastic.co/blog/how-to-ingest-geospatial-data-into-elasticsearch-with-gdal.
Thanks for getting back to me. I'll take a look at the links. If I can't get something working, I'll start learning js so I can use yours. 'enlightened laziness' lol
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Requires
elasticsearchandoboedependency.> npm install elasticsearch> npm install oboeExample command:
> node ./geojson_to_index.js boroughs.json boroughs geo_shape