From c2a4e0719226537460ddf82da60ea0d21bb81656 Mon Sep 17 00:00:00 2001 From: Jean Rouge Date: Mon, 2 May 2016 17:48:02 -0700 Subject: [PATCH] Some index changes: * making it possible to index without ID but with options * making it possible to bulk index without IDs * fixing bulk indexes which were broken if not using `jsx` * also accounting for the fact that `erlastic_json()` can also be a map * the only possibly breaking change is if someone out there used that lib to index a doc with the atom `undefined` as ID, which seems like a bad idea anyway (and is against the spec that it should be a binary) --- include/erlastic_search.hrl | 2 +- src/erlastic_search.erl | 49 +++++++++++++++++++++++-------------- src/erls_config.erl | 4 +-- 3 files changed, 33 insertions(+), 22 deletions(-) diff --git a/include/erlastic_search.hrl b/include/erlastic_search.hrl index 37247d7..0b40f4a 100644 --- a/include/erlastic_search.hrl +++ b/include/erlastic_search.hrl @@ -1,6 +1,6 @@ -type header() :: {binary(), binary()}. -type headers() :: [header()]. --type erlastic_json() :: tuple() | list(). +-type erlastic_json() :: tuple() | list() | map(). %% Hackney async references actually are just that, references... but it seems %% to be an undocumented implementation detail; doc (and specs) only says `any()' -type erlastic_success_result() :: erlastic_json() | {async, HackneyRef :: any()}. diff --git a/src/erlastic_search.erl b/src/erlastic_search.erl index 481301d..c299b07 100644 --- a/src/erlastic_search.erl +++ b/src/erlastic_search.erl @@ -20,6 +20,7 @@ ,put_mapping/4 ,index_doc/3 ,index_doc/4 + ,index_doc_with_opts/5 ,index_doc_with_id/4 ,index_doc_with_id/5 ,index_doc_with_id_opts/6 @@ -131,7 +132,7 @@ put_mapping(Index, Type, Doc) -> put_mapping(#erls_params{}, Index, Type, Doc). -spec put_mapping(#erls_params{}, binary(), binary(), erlastic_json() | binary()) -> {ok, erlastic_success_result()} | {error, any()}. -put_mapping(Params, Index, Type, Doc) when is_list(Doc); is_tuple(Doc) -> +put_mapping(Params, Index, Type, Doc) when is_list(Doc); is_tuple(Doc); is_map(Doc) -> put_mapping(Params, Index, Type, erls_json:encode(Doc)); put_mapping(Params, Index, Type, Doc) when is_binary(Doc) -> erls_resource:put(Params, filename:join([Index, Type, "_mapping"]), [], [], Doc, Params#erls_params.http_client_options). @@ -148,10 +149,14 @@ index_doc(Index, Type, Doc) -> index_doc(#erls_params{}, Index, Type, Doc). -spec index_doc(#erls_params{}, binary(), binary(), erlastic_json() | binary()) -> {ok, erlastic_success_result()} | {error, any()}. -index_doc(Params, Index, Type, Doc) when is_list(Doc); is_tuple(Doc) -> - index_doc(Params, Index, Type, erls_json:encode(Doc)); -index_doc(Params, Index, Type, Doc) when is_binary(Doc) -> - erls_resource:post(Params, filename:join(Index, Type), [], [], Doc, Params#erls_params.http_client_options). +index_doc(Params, Index, Type, Doc) -> + index_doc_with_opts(Params, Index, Type, Doc, []). + +-spec index_doc_with_opts(#erls_params{}, binary(), binary(), erlastic_json() | binary(), list()) -> {ok, erlastic_success_result()} | {error, any()}. +index_doc_with_opts(Params, Index, Type, Doc, Opts) when is_list(Opts), (is_list(Doc) orelse is_tuple(Doc) orelse is_map(Doc)) -> + index_doc_with_opts(Params, Index, Type, erls_json:encode(Doc), Opts); +index_doc_with_opts(Params, Index, Type, Doc, Opts) when is_list(Opts), is_binary(Doc) -> + erls_resource:post(Params, filename:join(Index, Type), [], Opts, Doc, Params#erls_params.http_client_options). %%-------------------------------------------------------------------- %% @doc @@ -169,9 +174,11 @@ index_doc_with_id(Params, Index, Type, Id, Doc) -> index_doc_with_id_opts(Params, Index, Type, Id, Doc, []). -spec index_doc_with_id_opts(#erls_params{}, binary(), binary(), binary(), erlastic_json() | binary(), list()) -> {ok, erlastic_success_result()} | {error, any()}. -index_doc_with_id_opts(Params, Index, Type, Id, Doc, Opts) when is_list(Opts), (is_list(Doc) orelse is_tuple(Doc)) -> +index_doc_with_id_opts(Params, Index, Type, undefined, Doc, Opts) -> + index_doc_with_opts(Params, Index, Type, Doc, Opts); +index_doc_with_id_opts(Params, Index, Type, Id, Doc, Opts) when is_list(Opts), (is_list(Doc) orelse is_tuple(Doc) orelse is_map(Doc)) -> index_doc_with_id_opts(Params, Index, Type, Id, erls_json:encode(Doc), Opts); -index_doc_with_id_opts(Params, Index, Type, Id, Doc, Opts) when is_binary(Doc), is_list(Opts) -> +index_doc_with_id_opts(Params, Index, Type, Id, Doc, Opts) when is_list(Opts), is_binary(Doc) -> erls_resource:post(Params, filename:join([Index, Type, Id]), [], Opts, Doc, Params#erls_params.http_client_options). @@ -189,7 +196,7 @@ upsert_doc(Params, Index, Type, Id, Doc) -> upsert_doc_opts(Params, Index, Type, Id, Doc, []). -spec upsert_doc_opts(#erls_params{}, binary(), binary(), binary(), erlastic_json(), list()) -> {ok, erlastic_success_result()} | {error, any()}. -upsert_doc_opts(Params, Index, Type, Id, Doc, Opts) when is_list(Opts), (is_list(Doc) orelse is_tuple(Doc)) -> +upsert_doc_opts(Params, Index, Type, Id, Doc, Opts) when is_list(Opts), (is_list(Doc) orelse is_tuple(Doc) orelse is_map(Doc)) -> DocBin = erls_json:encode(Doc), %% we cannot use erls_json to generate this, see the doc string for `erls_json:encode/1' Body = <<"{\"doc_as_upsert\":true,\"doc\":", DocBin/binary, "}">>, @@ -200,19 +207,12 @@ upsert_doc_opts(Params, Index, Type, Id, Doc, Opts) when is_list(Opts), (is_list %% Documents is [ {Index, Type, Id, Json}, {Index, Type, Id, HeaderInformation, Json}... ] -spec bulk_index_docs(#erls_params{}, list()) -> {ok, list()} | {error, any()}. bulk_index_docs(Params, IndexTypeIdJsonTuples) -> - MakeBody = fun - (Doc) when is_binary(Doc) -> - Doc; - (Doc) when is_list(Doc) orelse is_map(Doc) -> - erls_json:encode(Doc) - end, - Body = lists:map(fun Build({Index, Type, Id, Doc}) -> Build({Index, Type, Id, [], Doc}); Build({Index, Type, Id, HeaderInformation, Doc}) -> Header = bulk_index_docs_header(Index, Type, Id, HeaderInformation), - [ Header, <<"\n">>, MakeBody(Doc), <<"\n">> ] + [ Header, <<"\n">>, bulk_index_doc_body(Doc), <<"\n">> ] end, IndexTypeIdJsonTuples), erls_resource:post(Params, <<"/_bulk">>, [], [], iolist_to_binary(Body), Params#erls_params.http_client_options). @@ -355,9 +355,20 @@ commas([H | T]) -> -spec bulk_index_docs_header(binary(), binary(), binary(), list()) -> binary(). bulk_index_docs_header(Index, Type, Id, HeaderInformation) -> - erls_json:encode([{<<"index">>, [ + IndexHeaderJson1 = [ {<<"_index">>, Index} ,{<<"_type">>, Type} - ,{<<"_id">>, Id} | HeaderInformation - ]}]). \ No newline at end of file + ], + + IndexHeaderJson2 = case Id =:= undefined of + true -> [{<<"_id">>, Id} | IndexHeaderJson1]; + false -> IndexHeaderJson1 + end, + + %% we cannot use erls_json to generate this, see the doc string for `erls_json:encode/1' + jsx:encode([{<<"index">>, IndexHeaderJson2}]). + +-spec bulk_index_doc_body(binary() | erlastic_json()) -> binary(). +bulk_index_doc_body(Bin) when is_binary(Bin) -> Bin; +bulk_index_doc_body(Doc) -> erls_json:encode(Doc). diff --git a/src/erls_config.erl b/src/erls_config.erl index be50f87..83fba00 100644 --- a/src/erls_config.erl +++ b/src/erls_config.erl @@ -13,13 +13,13 @@ %% @doc %% Retrieves the default host. %% If nothing is defined in the app env for the key 'host', it's -%% <<"localhost">>. +%% <<"127.0.0.1">>. %% @end %%-------------------------------------------------------------------- get_host() -> case application:get_env(erlastic_search, host) of undefined -> - <<"localhost">>; + <<"127.0.0.1">>; {ok, Host}-> Host end.