Amazon OpenSearch Serviceを試してみる

$ mkdir opensearch-sample && cd opensearch-sample 
$ wget https://docs.aws.amazon.com/ja_jp/opensearch-service/latest/developerguide/samples/sample-movies.zip
$ unzip sample-movies.zip sample-movies.bulk

{ "index": { "_index": "movies", "_id": "tt2229499" } }
{"directors":["Joseph Gordon-Levitt"],"release_date":"2013-01-18T00:00:00Z","rating":7.4,"genres":["Comedy","Drama"],"image_url":"https://m.media-amazon.com/images/M/MV5BMTQxNTc3NDM2MF5BMl5BanBnXkFtZTcwNzQ5NTQ3OQ@@._V1_SX400_.jpg","plot":"A New Jersey guy dedicated to his family, friends, and church, develops unrealistic expectations from watching porn and works to find happiness and intimacy with his potential true love.","title":"Don Jon","rank":1,"running_time_secs":5400,"actors":["Joseph Gordon-Levitt","Scarlett Johansson","Julianne Moore"],"year":2013,"id":"tt2229499","type":"add"}
{ "index": { "_index": "movies", "_id": "tt1979320" } }
{"directors":["Ron Howard"],"release_date":"2013-09-02T00:00:00Z","rating":8.3,"genres":["Action","Biography","Drama","Sport"],"image_url":"https://m.media-amazon.com/images/M/MV5BMTQyMDE0MTY0OV5BMl5BanBnXkFtZTcwMjI2OTI0OQ@@._V1_SX400_.jpg","plot":"A re-creation of the merciless 1970s rivalry between Formula One rivals James Hunt and Niki Lauda.","title":"Rush","rank":2,"running_time_secs":7380,"actors":["Daniel Brühl","Chris Hemsworth","Olivia Wilde"],"year":2013,"id":"tt1979320","type":"add"}
{ "index": { "_index": "movies", "_id": "tt1392214" } }
(snip)

$ curl -X POST -u 'マスターユーザー名:マスターユーザーパスワード' 'OpenSearchドメインエンドポイントURL/_bulk' \
  -H 'Content-Type: x-ndjson' \
  --data-binary @sample-movies.bulk

{"error":{"root_cause":[{"type":"illegal_argument_exception","reason":"The bulk request must be terminated by a newline [\\n]"}],"type":"illegal_argument_exception","reason":"The bulk request must be terminated by a newline [\\n]"},"status":400}

$ tail -c 1 sample-movies.bulk | wc -l

$ curl -X POST -u 'マスターユーザー名:マスターユーザーパスワード' 'OpenSearchドメインエンドポイントURL/movies/_count'

{"count":5000,"_shards":{"total":5,"successful":5,"skipped":0,"failed":0}}

$ mkdir my-opensearch-function && cd my-opensearch-function
$ pip install --target ./package boto3
$ pip install --target ./package requests
$ pip install --target ./package requests_aws4auth 
$ cd package
$ zip -r ../my-deployment-package.zip .
$ cd ..

import boto3
import json
import requests
from requests_aws4auth import AWS4Auth

region = 'ap-northeast-1'
service = 'es'
credentials = boto3.Session().get_credentials()
awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, region, service, session_token=credentials.token)

host = 'OpenSearchドメインエンドポイントURL'  # "https://"含む、末尾"/"は含まない
index = 'movies'
url = host + '/' + index + '/_search'

# Lambda execution starts here
def lambda_handler(event, context):

    # Put the user query into the query DSL for more accurate search results.
    # Note that certain fields are boosted (^).
    query = {
        "size": 25,
        "query": {
            "multi_match": {
                "query": event['queryStringParameters']['q'],
                "fields": ["title^4", "plot^2", "actors", "directors"]
            }
        }
    }

    # Elasticsearch 6.x requires an explicit Content-Type header
    headers = { "Content-Type": "application/json" }

    # Make the signed HTTP request
    r = requests.get(url, auth=awsauth, headers=headers, data=json.dumps(query))

    # Create the response and add some extra content to support CORS
    response = {
        "statusCode": 200,
        "headers": {
            "Access-Control-Allow-Origin": '*'
        },
        "isBase64Encoded": False
    }

    # Add the search results to the response
    response['body'] = r.text
    return response

$ zip my-deployment-package.zip opensearch-lambda.py

$ curl https://Lambda関数URLのURL/?q=superman | jq

{
  "took": 4,
  "timed_out": false,
  "_shards": {
    "total": 5,
    "successful": 5,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": {
      "value": 7,
      "relation": "eq"
    },
    "max_score": 32.467476,
    "hits": [
      {
        "_index": "movies",
        "_id": "tt0078346",
        "_score": 32.467476,
        "_source": {
          "directors": [
            "Richard Donner"
          ],
          "release_date": "1978-12-10T00:00:00Z",
          "rating": 7.3,
          "genres": [
            "Action",
            "Adventure",
            "Fantasy",
            "Sci-Fi"
          ],
          "image_url": "https://m.media-amazon.com/images/M/MV5BMTI1MjA5MzM0OF5BMl5BanBnXkFtZTYwNTc0MTQ5._V1_SX400_.jpg",
          "plot": "An alien orphan is sent from his dying planet to Earth, where he grows up to become his adoptive home's first and greatest super-hero.",
          "title": "Superman",
          "rank": 970,
          "running_time_secs": 8580,
          "actors": [
            "Christopher Reeve",
            "Margot Kidder",
            "Gene Hackman"
          ],
          "year": 1978,
          "id": "tt0078346",
          "type": "add"
        }
      },
(snip)

var apigatewayendpoint = 'https://Lambda関数URLのURL';

[
  {
    "id": 575264,
    "title": "ミッション：インポッシブル／デッドレコニング PART ONE",
    "overview": "IMFエージェント、イーサン・ハントに課せられた(snip)",
    "genres": [
      "アクション",
      "XXXXXXX"
    ],
    "poster": "https://image.tmdb.org/t/p/w500/XXXXXXXXXXXXXXXXX.jpg",
    "release_date": 168XXXXXXX
  },
(snip)
]

{"index": {"_index": "movies_jp", "_id": "575264"}}
{"id": "575264", "title": "ミッション：インポッシブル／デッドレコニング PART ONE", "overview": "IMFエージェント、イーサン・ハントに課せられた(snip)", "genres": ["アクション", "XXXXX"], "poster": "https://image.tmdb.org/t/p/w500/XXXXXXXXXXXXXXXXX.jpg", "release_date": 168XXXXXXXXXX}
(snip)

PUT movies_jp
{
  "mappings": {
    "properties": {
      "id": {
        "type": "keyword"
      },
      "title": {
        "type": "text",
        "analyzer": "sudachi"
      },
      "overview": {
        "type": "text",
        "analyzer": "sudachi"
      },
      "genres": {
        "type": "keyword"
      },
      "poster": {
        "type": "keyword"
      },
      "release_date": {
        "type": "date",
        "format": "epoch_millis"
      }
    }
  }
}

$ curl -X POST -u 'マスターユーザー名:マスターユーザーパスワード' 'OpenSearchドメインエンドポイントURL/_bulk' \
  -H 'Content-Type: x-ndjson' \
  --data-binary @movies_jp.jsonl

(snip)
index = 'movies_jp'
(snip)
    query = {
        "size": 25,
        "query": {
            "multi_match": {
                "query": event['queryStringParameters']['q'],
                "fields": ["title^2", "overview"]
            }
        }
    }
(snip)

(snip)

function convertEpochToDateString(epochMillis) {
    var date = new Date(epochMillis);
    var year = date.getFullYear();
    var month = date.getMonth() + 1; // 月は0から始まるので+1する
    var day = date.getDate();
    // ゼロパディングしてYYYY-MM-DD形式にする
    return `${year}-${String(month).padStart(2, '0')}-${String(day).padStart(2, '0')}`;
}

async function search() {
  // Clear results before searching
  noresults.hide();
  resultdiv.empty();
  loadingdiv.show();
  // Get the query from the user
  let query = searchbox.val();
  // Only run a query if the string contains at least three characters
  if (query.length > 2) {
    // Make the HTTP request with the query as a parameter and wait for the JSON results
    let response = await $.get(apigatewayendpoint, { q: query, size: 25 }, 'json');
    // Get the part of the JSON response that we care about
    let results = response['hits']['hits'];
    if (results.length > 0) {
      loadingdiv.hide();
      // Iterate through the results and write them to HTML
      resultdiv.append('<p>Found ' + results.length + ' results.</p>');
      for (var item in results) {
        let url = 'https://www.themoviedb.org/movie/' + results[item]._id + '?language=ja';
        let image = results[item]._source.poster;
        let title = results[item]._source.title;
        let plot = results[item]._source.overview;
        let date = convertEpochToDateString(results[item]._source.release_date);
        // Construct the full HTML string that we want to append to the div
        resultdiv.append('<div class="result">' +
        '<a href="' + url + '"><img src="' + image + '" onerror="imageError(this)"></a>' +
        '<div><h2><a href="' + url + '">' + title + '</a></h2><p>' + date + ' &mdash; ' + plot + '</p></div></div>');
      }
    } else {
      noresults.show();
    }
  }
  loadingdiv.hide();
}
(snip)