json to csv.md

cat raw.response.from.kibana.json | jq .hits.hits[]._source > 1.json jq -s < 1.json > 2.json cat 2.json | jq '.[] + {"tags" : "_grokparsefailure_sysloginput"}' > 3.json jq -s < 3.json > 4.json cat 4.json | jq -r '(map(keys) | add | unique) as $cols | map(. as $row | $cols | map($row[.])) as $rows | $cols, $rows[] | @csv' > done.csv

url="http://localhost:9200"
# pid stands for pagination id.
pid=$(http -b POST "$url/hotel/_pit?keep_alive=10m" | jq .id)



base_request_body='{
  "size": 10000,
  "pit": {
	    "id":  %s, 
	    "keep_alive": "10m"
  },
    "sort": [
    {
      "hotel_name.keyword": {
        "order": "desc"
      }
    }
  ]
}
'


printf "$base_request_body" "$pid" | http -b GET "$url/_search" > 1st.10000.json

# making sure how many hits there is

hits=$(cat 1st.10000.json | jq '.hits.hits | length')
last_hit=$(echo "$hits - 1" | bc)
search_after=$(cat 1st.10000.json | jq .hits.hits[$last_hit].sort)

request_body_with_search_after='{
  "size": 10000,
  "pit": {
    "id": %s,
    "keep_alive": "10m"
  },
  "sort": [
    {
      "hotel_name.keyword": {
        "order": "desc"
      }
    }
  ],
  "search_after": %s
}' 



 printf "$request_body_with_search_after" "$pid" "$search_after" | http -b GET  "$url/_search" > 2nd.10000.json

TomonoriSoejima/json to csv.md