diff options
| -rwxr-xr-x | bin/build_and_unit_test.sh | 3 | ||||
| -rwxr-xr-x | get_version.sh | 2 | ||||
| -rw-r--r-- | rebar.config | 2 | ||||
| -rw-r--r-- | src/cdapbroker.app.src | 2 | ||||
| -rw-r--r-- | src/resource_handler.erl | 313 | ||||
| -rw-r--r-- | src/resource_handler_tests.erl | 136 | ||||
| -rw-r--r-- | swagger/swagger.html | 2 | ||||
| -rw-r--r-- | swagger/swagger.json | 2 | ||||
| -rw-r--r-- | swagger/swagger.yaml | 2 | ||||
| -rw-r--r-- | test/apitest/apitest_SUITE.erl | 143 |
10 files changed, 363 insertions, 244 deletions
diff --git a/bin/build_and_unit_test.sh b/bin/build_and_unit_test.sh index 7a2c757..9c3e3a4 100755 --- a/bin/build_and_unit_test.sh +++ b/bin/build_and_unit_test.sh @@ -2,4 +2,5 @@ rm -rf _build/; rebar3 upgrade; rebar3 release; -rebar3 eunit +rebar3 eunit --cover; +rebar3 cover diff --git a/get_version.sh b/get_version.sh index 3eba5a3..2624244 100755 --- a/get_version.sh +++ b/get_version.sh @@ -1,3 +1,3 @@ #!/bin/sh #todo... build this so that it parses automatically -echo "4.0.4 +echo "4.0.5 diff --git a/rebar.config b/rebar.config index e65730a..ae03f88 100644 --- a/rebar.config +++ b/rebar.config @@ -1,6 +1,6 @@ {relx, [ {release, - {cdapbroker,"4.0.4"}, + {cdapbroker,"4.0.5"}, [cdapbroker] }, %{extend_start_script,true}, diff --git a/src/cdapbroker.app.src b/src/cdapbroker.app.src index 157b141..b95d534 100644 --- a/src/cdapbroker.app.src +++ b/src/cdapbroker.app.src @@ -1,6 +1,6 @@ {application, cdapbroker, [{description, "Interface between Consul and CDAP in DCAE"}, - {vsn, "4.0.4"}, + {vsn, "4.0.5"}, {registered, []}, {mod, { cdapbroker_app, []}}, {applications, diff --git a/src/resource_handler.erl b/src/resource_handler.erl index a9703fc..8c1b343 100644 --- a/src/resource_handler.erl +++ b/src/resource_handler.erl @@ -39,13 +39,13 @@ get_request_id(Req) -> %ECOMP request tracing %see if we got a X-ECOMP-REQUESTID, or generate a new one if not HXER = leptus_req:header(Req, <<"x-ecomp-requestid">>), - case HXER of - undefined -> + case HXER of + undefined -> XER = util:gen_uuid(), %LOL, use the client ip here to shame them into their request id audit(warning, Req, [{bts, iso()}, {xer, XER}, {mod, mod()}, {msg, "Request is missing requestID. Assigned this one."}]), %eelf documentation says to log this message if requestid was missing XER; - _ -> + _ -> binary_to_list(HXER) %httpc expects strings as headers, so this needs to be str for subsequent passing end. @@ -58,24 +58,24 @@ init_api_call(Req) -> lookup_application(Appname) -> %do a lookup in mnesia of an appname Ret = mnesia:transaction(fun() -> mnesia:match_object(application, {application, Appname, '_', '_', '_', '_', '_', '_', '_', '_'}, read) end), - case Ret of + case Ret of {atomic, []} -> none; %no matches - {atomic, [Rec]} -> Rec + {atomic, [Rec]} -> Rec %fail hard if there was more than one result end. appname_to_application_map(Appname) -> %return a Map of an Mnesia record Rec = lookup_application(Appname), - case Rec of - none -> none; + case Rec of + none -> none; {application, Appname, AppType, Namespace, Healthcheckurl, Metricsurl, Url, Connectionurl, ServiceEndpoints, CreationTime} -> - #{<<"appname">> => Appname, + #{<<"appname">> => Appname, <<"apptype">> => AppType, <<"namespace">> => Namespace, - <<"healthcheckurl">> => Healthcheckurl, - <<"metricsurl">> => Metricsurl, - <<"url">> => Url, + <<"healthcheckurl">> => Healthcheckurl, + <<"metricsurl">> => Metricsurl, + <<"url">> => Url, <<"connectionurl">> => Connectionurl, <<"serviceendpoints">> => ServiceEndpoints, <<"creationtime">> => CreationTime @@ -85,7 +85,7 @@ appname_to_application_map(Appname) -> appname_to_field_vals(Appname, FieldList) -> %Return just a list of values of an application with fields FieldList M = appname_to_application_map(Appname), - case M of + case M of none -> none; _ -> [maps:get(F, M) || F <- FieldList] end. @@ -93,14 +93,14 @@ appname_to_field_vals(Appname, FieldList) -> appname_to_application_http(XER, Appname, State) -> %Return an HTTP response of an application record. If this is a program flowlet style app, additionally return it's bound and unbound config A = appname_to_application_map(Appname), - case A of - none -> {404, "", State}; - _ -> + case A of + none -> {404, "", State}; + _ -> Body = maps:with(?PUBLICFIELDS, A), case maps:get(<<"apptype">>, Body) of %if program-flowlet style app, append the bound and unbound config into the return JSON <<"program-flowlet">> -> - UB = case consul_interface:consul_get_configuration(XER, Appname, ?CONSURL) of + UB = case consul_interface:consul_get_configuration(XER, Appname, ?CONSURL) of {200, Unbound} -> Unbound; {_, _} -> <<"WARNING: COULD NOT FETCH CONFIG FROM CONSUL">> end, @@ -112,74 +112,11 @@ appname_to_application_http(XER, Appname, State) -> <<"bound_config">> => B}, {200, {json, maps:merge(Body, CM)}, State}; %TODO! can we do something for hydrator apps? - <<"hydrator-pipeline">> -> + <<"hydrator-pipeline">> -> {200, {json, Body}, State} end end. --spec parse_progflow_put_body_map(map()) -> - {binary(), binary(), string(), binary(), binary(), map(), map(), any(), lprogram(), any()}. %TODO! Spec parsedservices and parsedprogrampreferences so we don't have any() here... -parse_progflow_put_body_map(Body) -> - Namespace = maps:get(<<"namespace">>, Body), - Streamname = maps:get(<<"streamname">>, Body), - JarURL = maps:get(<<"jar_url">>, Body), - ArtifactName = maps:get(<<"artifact_name">>, Body), - ArtifactVersion = maps:get(<<"artifact_version">>, Body), - AppConfig = maps:get(<<"app_config">>, Body), - AppPreferences = maps:get(<<"app_preferences">>, Body), - ParsedServices = lists:map(fun(S) -> {maps:get(<<"service_name">>, S), - maps:get(<<"service_endpoint">>, S), - maps:get(<<"endpoint_method">>, S)} - end, maps:get(<<"services">>, Body)), - Programs = lists:map(fun(P) -> #program{type=maps:get(<<"program_type">>, P), - id= maps:get(<<"program_id">>, P)} - end, maps:get(<<"programs">>, Body)), - ParsedProgramPreferences = lists:map(fun(P) -> {maps:get(<<"program_type">>, P), - maps:get(<<"program_id">>, P), - maps:get(<<"program_pref">>, P)} - end, maps:get(<<"program_preferences">>, Body)), - {Namespace, Streamname, JarURL, ArtifactName, ArtifactVersion, AppConfig, AppPreferences, ParsedServices, Programs, ParsedProgramPreferences}. - -parse_hydrator_pipeline_put_body_map(Body) -> - Namespace = maps:get(<<"namespace">>, Body), - Streamname = maps:get(<<"streamname">>, Body), - PipelineConfigJsonURL = maps:get(<<"pipeline_config_json_url">>, Body), - - %Dependencies is optional. This function will normalize it's return with [] if the dependencies key was not passed in. - ParsedDependencies = case maps:is_key(<<"dependencies">>, Body) of - true -> - D = maps:get(<<"dependencies">>, Body), - %crash and let caller deal with it if not a list or if required keys are missing. Else parse it into - % {artifact-extends-header, artifact_name, artifact-version-header, artifact_url} - %tuples - % - %regarding the binart_to_lists: these all come in as binaries but they need to be "strings" (which are just lists of integers in erlang) - %for headers requiring strings, see http://stackoverflow.com/questions/28292576/setting-headers-in-a-httpc-post-request-in-erlang - % - lists:map(fun(X) -> {binary_to_list(maps:get(<<"artifact_extends_header">>, X)), - maps:get(<<"artifact_name">>, X), - binary_to_list(maps:get(<<"artifact_version_header">>, X)), - maps:get(<<"artifact_url">>, X), - %even if dependencies is specified, ui_properties is optional. This will normalize it's return with 'none' if not passed in - case maps:is_key(<<"ui_properties_url">>, X) of true -> maps:get(<<"ui_properties_url">>, X); false -> none end - } end, D); - false -> [] %normalize optional user input into []; just prevents user from having to explicitly pass in [] - end, - - {Namespace, Streamname, PipelineConfigJsonURL, ParsedDependencies}. - -parse_put_body(B) -> - Body = jiffy:decode(B, [return_maps]), - Type = maps:get(<<"cdap_application_type">>, Body), - case Type of - <<"program-flowlet">> -> - {pf, <<"program-flowlet">>, parse_progflow_put_body_map(Body)}; - <<"hydrator-pipeline">> -> - {hp, <<"hydrator-pipeline">>, parse_hydrator_pipeline_put_body_map(Body)}; - _ -> - unsupported - end. - delete_app_helper(Appname, State, XER, Req) -> %Helper because it is used by both delete and rollback on failed deploy % @@ -190,7 +127,7 @@ delete_app_helper(Appname, State, XER, Req) -> %1) Tell the user to try again later %2) Clean up as much as we can, log the error, and keep going % - %I have decided for now on taking number 2). This is the "Cloudify" way of doing things where you don't raise a NonRecoerable in a Delete operation. + %I have decided for now on taking number 2). This is the "Cloudify" way of doing things where you don't raise a NonRecoerable in a Delete operation. %This has the benefit that this delete operation can be used as the *rollback*, so if anything fails in the deploy, this delete function is called to clean up any dirty state. % %Number 1 is not so straitforward, because "putting back things the way they were" is difficult. For example, the deletion from CDAP succeeds, but Consul can't be reached. @@ -198,16 +135,16 @@ delete_app_helper(Appname, State, XER, Req) -> % %My conclusion is that transactions across distributed systems is hard. It's much easier if it is all local (e.g., Transactions in a single Postgres DB) % - %SO, as a result of this decision, the broker does *NOT* assert the status code of any delete operations to be 200. - %The only way this function does not return a 200 is if I can't even delete from my own database. + %SO, as a result of this decision, the broker does *NOT* assert the status code of any delete operations to be 200. + %The only way this function does not return a 200 is if I can't even delete from my own database. % - metrics(info, Req, [{bts, iso()}, {xer, XER}, {mod, mod()}, {msg, io_lib:format("Delete recieved for ~s", [Appname])}]), + metrics(info, Req, [{bts, iso()}, {xer, XER}, {mod, mod()}, {msg, io_lib:format("Delete recieved for ~s", [Appname])}]), case appname_to_field_vals(Appname, [<<"apptype">>, <<"namespace">>]) of none -> {404, "Tried to delete an application that was not registered", State}; [AppType, Namespace] -> try - case AppType of - <<"program-flowlet">> -> + case AppType of + <<"program-flowlet">> -> ok = workflows:undeploy_cdap_app(Req, XER, Appname, ?CDAPURL, ?CONSURL, Namespace), %delete from the program-flowlet supplementary table {atomic, ok} = mnesia:transaction(fun() -> mnesia:delete(prog_flow_supp, Appname, write) end); @@ -216,8 +153,8 @@ delete_app_helper(Appname, State, XER, Req) -> %delete from application table (shared between both types of apps) {atomic, ok} = mnesia:transaction(fun() -> mnesia:delete(application, Appname, write) end), {200, "", State} %Return - catch - %this is really bad, means I can't even delete from my own database. For now, log and pray. + catch + %this is really bad, means I can't even delete from my own database. For now, log and pray. %generic failure catch-all, catastrophic Class:Reason -> err(emergency, [{xer, XER}, {msg, io_lib:format("Catastrophic failure, can't delete ~s from my database. ~s:~s", [Appname, Class, Reason])}]), @@ -288,7 +225,7 @@ get("/application/:appname/healthcheck", Req, State) -> {Bts, XER} = init_api_call(Req), Appname = leptus_req:param(Req, appname), lager:info(io_lib:format("Get Healthcheck recieved for ~s", [Appname])), - {RCode, RBody, RState} = case appname_to_field_vals(Appname, [<<"apptype">>, <<"namespace">>]) of + {RCode, RBody, RState} = case appname_to_field_vals(Appname, [<<"apptype">>, <<"namespace">>]) of none -> {404, "", State}; [<<"program-flowlet">>, Namespace] -> {cdap_interface:get_app_healthcheck(XER, Appname, Namespace, ?CDAPURL), "", State}; @@ -305,22 +242,128 @@ delete("/application/:appname", Req, State) -> Appname = leptus_req:param(Req, appname), {RCode, RBody, RState} = delete_app_helper(Appname, State, XER, Req), ?AUDI(Req, Bts, XER, Rcode), - {RCode, RBody, RState}. + {RCode, RBody, RState}. %%%PUT Methods +parse_put_body(B) -> + %parse the PUT body to application + try + Body = jiffy:decode(B, [return_maps]), + Type = maps:get(<<"cdap_application_type">>, Body), + case Type == <<"hydrator-pipeline">> orelse Type == <<"program-flowlet">> of + false -> unsupported; + true -> + %common to both + Namespace = maps:get(<<"namespace">>, Body), + Streamname = maps:get(<<"streamname">>, Body), + case Type of + <<"program-flowlet">> -> + JarURL = maps:get(<<"jar_url">>, Body), + ArtifactName = maps:get(<<"artifact_name">>, Body), + ArtifactVersion = maps:get(<<"artifact_version">>, Body), + AppConfig = maps:get(<<"app_config">>, Body), + AppPreferences = maps:get(<<"app_preferences">>, Body), + ParsedServices = lists:map(fun(S) -> {maps:get(<<"service_name">>, S), + maps:get(<<"service_endpoint">>, S), + maps:get(<<"endpoint_method">>, S)} + end, maps:get(<<"services">>, Body)), + Programs = lists:map(fun(P) -> #program{type=maps:get(<<"program_type">>, P), + id= maps:get(<<"program_id">>, P)} + end, maps:get(<<"programs">>, Body)), + ParsedProgramPreferences = lists:map(fun(P) -> {maps:get(<<"program_type">>, P), + maps:get(<<"program_id">>, P), + maps:get(<<"program_pref">>, P)} + end, maps:get(<<"program_preferences">>, Body)), + {pf, <<"program-flowlet">>, {Namespace, Streamname, JarURL, ArtifactName, ArtifactVersion, AppConfig, AppPreferences, ParsedServices, Programs, ParsedProgramPreferences}}; + <<"hydrator-pipeline">> -> + PipelineConfigJsonURL = maps:get(<<"pipeline_config_json_url">>, Body), + + %Dependencies is optional. This function will normalize it's return with [] if the dependencies key was not passed in. + ParsedDependencies = case maps:is_key(<<"dependencies">>, Body) of + true -> + D = maps:get(<<"dependencies">>, Body), + %crash and let caller deal with it if not a list or if required keys are missing. Else parse it into + % {artifact-extends-header, artifact_name, artifact-version-header, artifact_url} + %regarding the binart_to_lists: these all come in as binaries but they need to be "strings" (which are just lists of integers in erlang) + %for headers requiring strings, see http://stackoverflow.com/questions/28292576/setting-headers-in-a-httpc-post-request-in-erlang + lists:map(fun(X) -> {binary_to_list(maps:get(<<"artifact_extends_header">>, X)), + maps:get(<<"artifact_name">>, X), + binary_to_list(maps:get(<<"artifact_version_header">>, X)), + maps:get(<<"artifact_url">>, X), + %even if dependencies is specified, ui_properties is optional. This will normalize it's return with 'none' if not passed in + case maps:is_key(<<"ui_properties_url">>, X) of true -> maps:get(<<"ui_properties_url">>, X); false -> none end + } end, D); + false -> [] %normalize optional user input into []; just prevents user from having to explicitly pass in [] + end, + {hp, <<"hydrator-pipeline">>, {Namespace, Streamname, PipelineConfigJsonURL, ParsedDependencies}} + end + end + catch _:_ -> invalid + end. + +parse_reconfiguration_put_body(Body) -> + try + D = jiffy:decode(Body, [return_maps]), + ReconfigurationType = maps:get(<<"reconfiguration_type">>, D), + Config = maps:get(<<"config">>, D), + case ReconfigurationType == <<"program-flowlet-app-config">> orelse + ReconfigurationType == <<"program-flowlet-app-preferences">> orelse + ReconfigurationType == <<"program-flowlet-smart">> of + false -> notimplemented; + true -> {ReconfigurationType, Config} + end + catch _:_ -> invalid + end. + +handle_reconfigure_put(Req, State, XER, Appname, ReqBody, AppnameToNS) -> + %handle the reconfiguration put. broker out from the http call, and takes the lookup func as an arg, to allow for better unit testing. + %this is still not a pure function due to the workflows call, still needs enhancement + case AppnameToNS(Appname) of + none -> {404, "Reconfigure recieved but the app is not registered", State}; + Namespace -> + ParsedBody = parse_reconfiguration_put_body(ReqBody), + case ParsedBody of + invalid -> {400, "Invalid PUT Reconfigure Body", State}; + notimplemented -> {501, "This type of reconfiguration is not implemented", State}; + {ReconfigurationType, Config} -> + try + ok = case ReconfigurationType of + <<"program-flowlet-app-config">> -> + %reconfigure a program-flowlet style app's app config + workflows:app_config_reconfigure(Req, XER, Appname, Namespace, ?CONSURL, ?CDAPURL, Config); + <<"program-flowlet-app-preferences">> -> + %reconfigure a program-flowlet style app's app config + workflows:app_preferences_reconfigure(Req, XER, Appname, Namespace, ?CONSURL, ?CDAPURL, Config); + <<"program-flowlet-smart">> -> + %try to "figure out" whether the supplied JSON contains keys in appconfig, app preferences, or both + workflows:smart_reconfigure(Req, XER, Appname, Namespace, ?CONSURL, ?CDAPURL, Config) + end, + {200, "", State} + catch + %catch a bad HTTP error code; also catches the non-overlapping configuration case + error:{badmatch, {BadErrorCode, BadStatusMsg}} -> + err(error, [{xer, XER}, {msg, io_lib:format("~p ~s", [BadErrorCode, BadStatusMsg])}]), + {BadErrorCode, BadStatusMsg, State}; + Class:Reason -> + err(error, [{xer,XER}, {msg, io_lib:format("~nError Stacktrace:~s", [lager:pr_stacktrace(erlang:get_stacktrace(), {Class, Reason})])}]), + {500, "", State} + end + end + end. + put("/application/:appname", Req, State) -> %create a new registration; deploys and starts a cdap application {Bts, XER} = init_api_call(Req), Appname = leptus_req:param(Req, appname), {RCode, RBody, RState} = case appname_to_field_vals(Appname, [<<"appname">>]) of - [Appname] -> + [Appname] -> {400, "Put recieved on /application/:appname but appname is already registered. Call /application/:appname/reconfigure if trying to reconfigure or delete first", State}; none -> %no matches, create the resource, return the application record %Initial put requires the put body parameters - case try parse_put_body(leptus_req:body_raw(Req)) catch _:_ -> invalid end of + case parse_put_body(leptus_req:body_raw(Req)) of %could not parse the body invalid -> {400, "Invalid PUT Body or unparseable URL", State}; - + %unsupported cdap application type unsupported -> {404, "Unsupported CDAP Application Type", State}; @@ -330,13 +373,13 @@ put("/application/:appname", Req, State) -> {RequestUrl,_} = cowboy_req:url((leptus_req:get_req(Req))), Metricsurl = <<RequestUrl/binary, <<"/metrics">>/binary>>, Healthcheckurl = <<RequestUrl/binary, <<"/healthcheck">>/binary>>, - + try - case Type of - hp -> + case Type of + hp -> {Namespace, Streamname, PipelineConfigJsonURL, ParsedDependencies} = Params, ConnectionURL = cdap_interface:form_stream_url_from_streamname(?CDAPURL, Namespace, Streamname), - + %TODO: This! ServiceEndpoints = [], %unclear if this is possible with pipelines @@ -350,21 +393,21 @@ put("/application/:appname", Req, State) -> {Namespace, Streamname, JarURL, ArtifactName, ArtifactVersion, AppConfig, AppPreferences, ParsedServices, Programs, ParsedProgramPreferences} = Params, %Form URLs that are part of the record %NOTE: These are both String concatenation functions and neither make an HTTP call so not catching normal {Code, Status} return here - ConnectionURL = cdap_interface:form_stream_url_from_streamname(?CDAPURL, Namespace, Streamname), + ConnectionURL = cdap_interface:form_stream_url_from_streamname(?CDAPURL, Namespace, Streamname), ServiceEndpoints = lists:map(fun(X) -> cdap_interface:form_service_json_from_service_tuple(Appname, Namespace, ?CDAPURL, X) end, ParsedServices), - + %write into mnesia. deploy A = #application{appname = Appname, apptype = AppType, namespace = Namespace, healthcheckurl = Healthcheckurl, metricsurl = Metricsurl, url = RequestUrl, connectionurl = ConnectionURL, serviceendpoints = ServiceEndpoints, creationtime=erlang:system_time()}, ASupplemental = #prog_flow_supp{appname = Appname, programs = Programs}, {atomic,ok} = mnesia:transaction(fun() -> mnesia:write(A) end), %warning, here be mnesia magic that knows what table you want to write to based on the record type {atomic,ok} = mnesia:transaction(fun() -> mnesia:write(ASupplemental) end), %warning: "" ok = workflows:deploy_cdap_app(Req, XER, Appname, ?CONSURL, ?CDAPURL, ?HCInterval, ?AutoDeregisterAfter, AppConfig, JarURL, ArtifactName, ArtifactVersion, Namespace, AppPreferences, ParsedProgramPreferences, Programs, RequestUrl, Healthcheckurl), - metrics(info, Req, [{bts, iso()}, {xer, XER}, {mod, mod()}, {msg, io_lib:format("New Program-Flowlet Application Created: ~p with supplemental data: ~p", [lager:pr(A, ?MODULE), lager:pr(ASupplemental, ?MODULE)])}]), + metrics(info, Req, [{bts, iso()}, {xer, XER}, {mod, mod()}, {msg, io_lib:format("New Program-Flowlet Application Created: ~p with supplemental data: ~p", [lager:pr(A, ?MODULE), lager:pr(ASupplemental, ?MODULE)])}]), ok end, appname_to_application_http(XER, Appname, State) - - catch + + catch %catch a bad HTTP error code error:{badmatch, {BadErrorCode, BadStatusMsg}} -> err(error, [{xer, XER}, {msg, io_lib:format("Badmatch caught in Deploy. Rolling Back. ~p ~s", [BadErrorCode, BadStatusMsg])}]), @@ -384,53 +427,15 @@ put("/application/:appname/reconfigure", Req, State) -> %if appname already is registerd, trigger a consul pull and reconfigure {Bts, XER} = init_api_call(Req), Appname = leptus_req:param(Req, appname), - {RCode, RBody, RState} = case appname_to_field_vals(Appname, [<<"namespace">>]) of - none -> {404, "Reconfigure recieved but the app is not registered", State}; - [Namespace] -> - D = jiffy:decode(leptus_req:body_raw(Req), [return_maps]), - case try maps:get(<<"config">>, D) catch _:_ -> invalid end of - invalid -> {400, "Invalid PUT Reconfigure Body: key 'config' is missing", State}; - Config -> - case try maps:get(<<"reconfiguration_type">>, D) catch _:_ -> invalid end of - invalid -> {400, "Invalid PUT Reconfigure Body: key 'reconfiguration_type' is missing", State}; - <<"program-flowlet-app-config">> -> - %reconfigure a program-flowlet style app's app config - try - ok = workflows:app_config_reconfigure(Req, XER, Appname, Namespace, ?CONSURL, ?CDAPURL, Config), - {200, "", State} - catch Class:Reason -> - err(error, [{xer,XER}, {msg, io_lib:format("~nError Stacktrace:~s", [lager:pr_stacktrace(erlang:get_stacktrace(), {Class, Reason})])}]), - {500, "", State} - end; - <<"program-flowlet-app-preferences">> -> - %reconfigure a program-flowlet style app's app config - try - ok = workflows:app_preferences_reconfigure(Req, XER, Appname, Namespace, ?CONSURL, ?CDAPURL, Config), - {200, "", State} - catch Class:Reason -> - err(error, [{xer,XER}, {msg, io_lib:format("~nError Stacktrace:~s", [lager:pr_stacktrace(erlang:get_stacktrace(), {Class, Reason})])}]), - {500, "", State} - end; - <<"program-flowlet-smart">> -> - %try to "figure out" whether the supplied JSON contains keys in appconfig, app preferences, or both - try - ok = workflows:smart_reconfigure(Req, XER, Appname, Namespace, ?CONSURL, ?CDAPURL, Config), - {200, "", State} - catch - %catch a bad HTTP error code; also catches the non-overlapping configuration case - error:{badmatch, {BadErrorCode, BadStatusMsg}} -> - err(error, [{xer, XER}, {msg, io_lib:format("~p ~s", [BadErrorCode, BadStatusMsg])}]), - {BadErrorCode, BadStatusMsg, State}; - Class:Reason -> - err(error, [{xer,XER}, {msg, io_lib:format("~nError Stacktrace:~s", [lager:pr_stacktrace(erlang:get_stacktrace(), {Class, Reason})])}]), - {500, "", State} - end; - NI -> - %TODO! Implement other types of reconfig once CDAP APIs exis - {501, io_lib:format("This type (~s) of reconfiguration is not implemented", [NI]), State} - end - end - end, + ReqBody = leptus_req:body_raw(Req), + AppnameToNS = fun(App) -> + X = appname_to_field_vals(App, [<<"namespace">>]), + case X of + none -> X; + [Namespace] -> Namespace + end + end, + {RCode, RBody, RState} = handle_reconfigure_put(Req, State, XER, Appname, ReqBody, AppnameToNS), ?AUDI(Req, Bts, XER, Rcode), {RCode, RBody, RState}. @@ -439,20 +444,20 @@ post("/application/delete", Req, State) -> %This follows the AWS S3 Multi Key Delete: http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html %Except I added an additional special value called "*" {Bts, XER} = init_api_call(Req), - {RCode, RBody, RState} = case try + {RCode, RBody, RState} = case try B = maps:get(<<"appnames">>, jiffy:decode(leptus_req:body_raw(Req), [return_maps])), true = erlang:is_list(B), B - catch _:_ -> - invalid - end + catch _:_ -> + invalid + end of invalid -> {400, "Invalid PUT Body", State}; IDs -> case IDs of [] -> {200, "EMPTY PUT BODY", State}; _ -> - %<<"*">> -> + %<<"*">> -> %this block deleted all apps, but decided this backdoor wasn't very RESTy %% {atomic, Apps} = mnesia:transaction(fun() -> mnesia:match_object(application, {application, '_', '_', '_', '_', '_', '_', '_', '_', '_'}, read) end), % AppsToDelete = lists:map(fun(X) -> {application, Appname, _,_,_,_,_,_,_,_} = X, Appname end, Apps), diff --git a/src/resource_handler_tests.erl b/src/resource_handler_tests.erl new file mode 100644 index 0000000..cab9558 --- /dev/null +++ b/src/resource_handler_tests.erl @@ -0,0 +1,136 @@ +% ============LICENSE_START======================================================= +% org.onap.dcae +% ================================================================================ +% Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +% ================================================================================ +% Licensed under the Apache License, Version 2.0 (the "License"); +% you may not use this file except in compliance with the License. +% You may obtain a copy of the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, +% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +% See the License for the specific language governing permissions and +% limitations under the License. +% ============LICENSE_END========================================================= +% +% ECOMP is a trademark and service mark of AT&T Intellectual Property. + +-module(resource_handler_tests). +-include_lib("eunit/include/eunit.hrl"). +-include("application.hrl"). +-import(resource_handler, [ + parse_put_body/1, + parse_reconfiguration_put_body/1, + handle_reconfigure_put/6 + ]). + +parse_put_body_test() -> + Valid = {[ + {<<"cdap_application_type">>, <<"program-flowlet">>}, + {<<"namespace">>, <<"ns">>}, + {<<"streamname">>, <<"sn">>}, + {<<"jar_url">>, <<"www.foo.com">>}, + {<<"artifact_name">>, <<"art_name">>}, + {<<"artifact_version">>, <<"art_ver">>}, + {<<"app_config">>, {[{<<"foo">>,<<"bar">>}]}}, + {<<"app_preferences">>, {[{<<"foop">>,<<"barp">>}]}}, + {<<"services">>, [{[{<<"service_name">>, <<"Greeting">>}, + {<<"service_endpoint">>, <<"greet">>}, + {<<"endpoint_method">>, <<"GET">>}]}]}, + {<<"programs">>, [ + {[{<<"program_type">>, <<"flows">>}, + {<<"program_id">>, <<"WhoFlow">>}]}, + {[{<<"program_type">>, <<"services">>}, + {<<"program_id">>, <<"Greeting">>}]}]}, + {<<"program_preferences">>, [ + {[{<<"program_type">>,<<"flows">>}, + {<<"program_id">>, <<"WhoFlow">>}, + {<<"program_pref">>, {[{<<"foopprog">>,<<"barpprog">>}]}}]} + ]} + ]}, + %{Namespace, Streamname, JarURL, ArtifactName, ArtifactVersion, AppConfig, AppPreferences, ParsedServices, Programs, ParsedProgramPreferences} + ExpectedL = {<<"ns">>, <<"sn">>, <<"www.foo.com">>, <<"art_name">>, <<"art_ver">>, + #{<<"foo">>=><<"bar">>}, + #{<<"foop">>=><<"barp">>}, + [{<<"Greeting">>,<<"greet">>,<<"GET">>}], + [#program{type = <<"flows">>, id = <<"WhoFlow">>}, #program{type = <<"services">>, id = <<"Greeting">>}], + [{<<"flows">>,<<"WhoFlow">>,#{<<"foopprog">>=><<"barpprog">>}}]}, + Expected = {pf, <<"program-flowlet">>, ExpectedL}, + ?assert(parse_put_body(jiffy:encode(Valid)) == Expected), + + ValidHydrator1 = + {[ + {<<"cdap_application_type">>, <<"hydrator-pipeline">>}, + {<<"namespace">>, <<"ns">>}, + {<<"streamname">>, <<"sn">>}, + {<<"pipeline_config_json_url">>, "www.foo.com"} + ]}, + ExpectedHy1 = {hp,<<"hydrator-pipeline">>,{<<"ns">>,<<"sn">>,"www.foo.com",[]}}, + ?assert(parse_put_body(jiffy:encode(ValidHydrator1)) == ExpectedHy1), + + ValidHydrator2 = + {[ + {<<"cdap_application_type">>, <<"hydrator-pipeline">>}, + {<<"namespace">>, <<"ns">>}, + {<<"streamname">>, <<"sn">>}, + {<<"pipeline_config_json_url">>, "www.foo.com"}, + {<<"dependencies">>, [ + {[ + {<<"artifact_extends_header">>, <<"system:cdap-data-pipeline[4.1.0,5.0.0)">>}, + {<<"artifact_name">>, <<"art carney">>}, + {<<"artifact_version_header">>, <<"1.0.0-SNAPSHOT">>}, + {<<"artifact_url">>, <<"www.foo.com/sup/baphomet.jar">>}, + {<<"ui_properties_url">>, <<"www.foo2.com/sup/baphomet.jar">>} + ]} + ]} + ]}, + %{hp, <<"hydrator-pipeline">>, {Namespace, Streamname, PipelineConfigJsonURL, ParsedDependencies}} + ExpectedHy2 = {hp,<<"hydrator-pipeline">>,{<<"ns">>,<<"sn">>,"www.foo.com",[{"system:cdap-data-pipeline[4.1.0,5.0.0)",<<"art carney">>,"1.0.0-SNAPSHOT",<<"www.foo.com/sup/baphomet.jar">>,<<"www.foo2.com/sup/baphomet.jar">>}]}}, + ?assert(parse_put_body(jiffy:encode(ValidHydrator2)) == ExpectedHy2), + + InvalidType = {[{<<"cdap_application_type">>, <<"NOT TODAY">>}]}, + erlang:display(parse_put_body(jiffy:encode(InvalidType))), + ?assert(parse_put_body(jiffy:encode(InvalidType)) == unsupported), + + InvalidMissing = {[ + {<<"cdap_application_type">>, <<"program-flowlet">>}, + {<<"namespace">>, <<"ns">>} + ]}, + ?assert(parse_put_body(jiffy:encode(InvalidMissing)) == invalid). + +reconfiguration_put_test() -> + %test reconfiguring with an invalid PUT body (missing "reconfiguration_type") + AppnameToNS = fun(X) -> + case X of + <<"notexist">> -> none; + <<"exist">> -> <<"ns">> + end + end, + EmptyD = dict:new(), + + I1 = jiffy:encode({[{<<"config">>, <<"bar">>}]}), + ?assert(parse_reconfiguration_put_body(I1) == invalid), + ?assert(handle_reconfigure_put("", EmptyD, "testXER", <<"exist">>, I1, AppnameToNS) == {400,"Invalid PUT Reconfigure Body",EmptyD}), + + %test reconfiguring with an invalid PUT body (missing app_config) + I2 = jiffy:encode({[{<<"reconfiguration_type">>, <<"program-flowlet-app-config">>}, {<<"foo">>, <<"bar">>}]}), + ?assert(parse_reconfiguration_put_body(I2) == invalid), + ?assert(handle_reconfigure_put("", EmptyD, "testXER", <<"exist">>, I2, AppnameToNS) == {400,"Invalid PUT Reconfigure Body",EmptyD}), + + %test reconfiguring an invalid (unimplemented) type + I3 = jiffy:encode({[{<<"config">>, <<"bar">>}, {<<"reconfiguration_type">>, <<"EMPTINESS">>}]}), + ?assert(parse_reconfiguration_put_body(I3) == notimplemented), + ?assert(handle_reconfigure_put("", EmptyD, "testXER", <<"exist">>, I3, AppnameToNS) == {501,"This type of reconfiguration is not implemented",EmptyD}), + + Valid = jiffy:encode({[{<<"config">>, {[{<<"foo">>, <<"bar">>}]}}, {<<"reconfiguration_type">>,<<"program-flowlet-app-config">>}]}), + ?assert(parse_reconfiguration_put_body(Valid) == {<<"program-flowlet-app-config">>,#{<<"foo">>=><<"bar">>}}), + + %no unit test for handle_reconfigure_put yet + + %test for valid but missing + %?assert(handle_reconfigure_put("", EmptyD, "testXER", <<"exist">>, I3, AppnameToNS) == {501,"This type of reconfiguration is not implemented",EmptyD}), + ?assert(handle_reconfigure_put("", EmptyD, "testXER", <<"notexist">>, Valid, AppnameToNS) == {404,"Reconfigure recieved but the app is not registered", EmptyD}). + diff --git a/swagger/swagger.html b/swagger/swagger.html index 0cce633..80b2176 100644 --- a/swagger/swagger.html +++ b/swagger/swagger.html @@ -12,7 +12,7 @@ <body> <div class="container"> <h1>CDAP Broker API</h1> - <p class="sw-info">Version: <span class="sw-info-version">4.0.4</span></p> + <p class="sw-info">Version: <span class="sw-info-version">4.0.5</span></p> <p></p> diff --git a/swagger/swagger.json b/swagger/swagger.json index fbfd932..f1b1480 100644 --- a/swagger/swagger.json +++ b/swagger/swagger.json @@ -1,7 +1,7 @@ { "swagger": "2.0", "info": { - "version": "4.0.4", + "version": "4.0.5", "title": "CDAP Broker API" }, "paths": { diff --git a/swagger/swagger.yaml b/swagger/swagger.yaml index ed8fd68..a7d14f8 100644 --- a/swagger/swagger.yaml +++ b/swagger/swagger.yaml @@ -5,7 +5,7 @@ swagger: '2.0' # This is your document metadata info: - version: "4.0.4" + version: "4.0.5" title: CDAP Broker API paths: diff --git a/test/apitest/apitest_SUITE.erl b/test/apitest/apitest_SUITE.erl index 971204c..1f7a3b2 100644 --- a/test/apitest/apitest_SUITE.erl +++ b/test/apitest/apitest_SUITE.erl @@ -6,9 +6,9 @@ % Licensed under the Apache License, Version 2.0 (the "License"); % you may not use this file except in compliance with the License. % You may obtain a copy of the License at -% +% % http://www.apache.org/licenses/LICENSE-2.0 -% +% % Unless required by applicable law or agreed to in writing, software % distributed under the License is distributed on an "AS IS" BASIS, % WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,7 +21,7 @@ -include_lib("common_test/include/ct.hrl"). -include("../../src/application.hrl"). -export([all/0, groups/0, init_per_suite/1, end_per_suite/1]). --export([server_health_test/1, app_deploy/1, hydrator_deploy/1, app_teardown/1, app_test/1, app_reconfigure/1, test_failures/1, app_botch_flows/1, app_botch_delete/1, app_botch_consul_delete/1, invalid_reconfigure/1, delete_all/1, +-export([server_health_test/1, app_deploy/1, hydrator_deploy/1, app_teardown/1, app_test/1, app_reconfigure/1, test_failures/1, app_botch_flows/1, app_botch_delete/1, app_botch_consul_delete/1, delete_all/1, hydrator_app_teardown/1, hydrator_test/1, hydrator_wdeps_deploy/1, hydrator_wdeps_test/1, @@ -30,7 +30,7 @@ %lazy shorthands (yay C style macros! miss these in python) -define(SC(L), util:concat(L)). --define(PLG(K, PL), proplists:get_value(K, PL)). +-define(PLG(K, PL), proplists:get_value(K, PL)). -define(XER, "testing-XER"). -define(D(X), erlang:display(X)). @@ -40,7 +40,6 @@ all() -> [ {group, apibotchedflows}, {group, apibotcheddeleted}, {group, apibotchedconsuldeleted}, - {group, invalidreconfig}, {group, apideleteall} ]. groups() -> [ @@ -91,15 +90,7 @@ groups() -> [ app_botch_consul_delete, app_teardown ]}, - {invalidreconfig, %call reconfigure on an app that DNE - [], - [ - server_health_test, - app_deploy, - invalid_reconfigure, - app_teardown - ]}, - {apideleteall, + {apideleteall, [], [ server_health_test, @@ -119,9 +110,9 @@ setup_rels(Config, D) -> % 4 Broker binds config % 5 Broker pushes bound config to CDAP % Between state 1 and 3 consul is in an inconsistent state where it has only the rels key but not the config key. Not so sure about this. They seem to be a pair. Maybe the rels key should be pushed to the source node to be dealt with. - % #Here, we are mocking step 1 + % #Here, we are mocking step 1 URL = ?SC([?PLG(consul_url, Config), "/v1/kv/", ?PLG(appname, Config), ":rel"]), - case D of + case D of setup -> {200,"true"} = httpabs:put(?XER, URL, "application/json", jiffy:encode([<<"666_fake_testing_service">>])); teardown -> {200,"true"} = httpabs:delete(?XER, URL) end, @@ -131,7 +122,7 @@ setup_fake_testing_service(Config, D) -> %register a fake testing service to test that the CDAP app recieved it's bound configuration properly Name = <<"666_fake_testing_service">>, SrvURL = ?SC([?PLG(consul_url, Config), "/v1/catalog/service/", Name]), - case D of + case D of setup -> URL = ?SC([?PLG(consul_url, Config), "/v1/agent/service/register"]), Body = {[{<<"name">>, Name}, @@ -147,10 +138,10 @@ setup_fake_testing_service(Config, D) -> httpabs:get(?XER, SrvURL) end. -get_config_consul(C) -> +get_config_consul(C) -> %get config from consul. returns the code too for tests testing for a 404 - {RC, RB} = consul_interface:consul_get_configuration(?XER, ?PLG(appname, C), ?PLG(consul_url, C)), - case RC of + {RC, RB} = consul_interface:consul_get_configuration(?XER, ?PLG(appname, C), ?PLG(consul_url, C)), + case RC of 200 -> {RC, util:ejson_to_map(RB)}; _ -> {RC, RB} end. @@ -158,7 +149,7 @@ get_config_consul(C) -> get_config_cdap(C) -> {RC, RB} = cdap_interface:get_app_config(?XER, ?PLG(appname, C), ?PLG(namespace, C),?PLG(cdap_url, C)), case RC of - 200 -> + 200 -> %I think CDAP is DOUBLY encoding JSON!! {RC, jiffy:decode(jiffy:decode(jiffy:encode(RB)), [return_maps])}; _ -> {RC, RB} @@ -171,10 +162,10 @@ get_preferences_cdap(C) -> _ -> {RC, RB} end. -get_preferences_consul(C) -> +get_preferences_consul(C) -> %get preferences from consul. returns the code too for tests testing for a 404 {RC, RB} = consul_interface:consul_get_preferences(?XER, ?PLG(appname, C), ?PLG(consul_url, C)), - case RC of + case RC of 200 -> {RC, util:ejson_to_map(RB)}; _ -> {RC, RB} end. @@ -189,7 +180,7 @@ valid_deploy_body(C) -> {<<"artifact_version">>, ?PLG(art_ver, C)}, {<<"app_config">>, ?PLG(init_config, C)}, {<<"app_preferences">>, ?PLG(init_preferences, C)}, - {<<"services">>, [{[{<<"service_name">>, <<"Greeting">>}, + {<<"services">>, [{[{<<"service_name">>, <<"Greeting">>}, {<<"service_endpoint">>, <<"greet">>}, {<<"endpoint_method">>, <<"GET">>}]}]}, {<<"programs">>, [ @@ -198,8 +189,8 @@ valid_deploy_body(C) -> {[{<<"program_type">>, <<"services">>}, {<<"program_id">>, <<"Greeting">>}]}]}, {<<"program_preferences">>, [ - {[{<<"program_type">>,<<"flows">>}, - {<<"program_id">>, <<"WhoFlow">>}, + {[{<<"program_type">>,<<"flows">>}, + {<<"program_id">>, <<"WhoFlow">>}, {<<"program_pref">>, ?PLG(whoflowpref, C)}]} ]} ]}. @@ -209,7 +200,7 @@ valid_deploy_body(C) -> init_per_suite(_C) -> %get platform ENVs [MyName, ConsulURL, _, _] = util:get_platform_envs_and_config(), - + BrokerUrl = case os:getenv("BROKER_TEST_TYPE") of false -> %no env variable means start the broker on localhost %start a local broker @@ -229,7 +220,7 @@ init_per_suite(_C) -> {200, RB} = httpabs:get(?XER, BrokerUrl), CDAPUrl = maps:get(<<"managed cdap url">>, jiffy:decode(RB, [return_maps])), - + %set properties that are shared between program-flowlet and hydrator Namespace = <<"testns">>, CDAPUrlNS = ?SC([CDAPUrl, "/v3/namespaces/", Namespace]), @@ -242,14 +233,14 @@ init_per_suite(_C) -> HydratorAppname = <<"hydratortest">>, HydratorAppURL = ?SC([CDAPUrlNS, "/apps/", HydratorAppname]), HydratorStreamname = <<"s1">>, %horrible name but not made by me - + HydratorWDepsAppname = <<"hydratorwdepstest">>, HydratorWDepsAppURL = ?SC([CDAPUrlNS, "/apps/", HydratorWDepsAppname]), HydratorWDepsStreamname = <<"t1">>, %horrible name but not made by me - %Set up this test suites configuration - [{broker_url, BrokerUrl}, - {cdap_url, CDAPUrl}, + %Set up this test suites configuration + [{broker_url, BrokerUrl}, + {cdap_url, CDAPUrl}, {cdap_ns_url, CDAPUrlNS}, {jar_url, ?SC([Nexus, "/jar_files/HelloWorld-3.4.3.jar"])}, {consul_url, ConsulURL}, @@ -314,7 +305,7 @@ server_health_test(C) -> app_deploy(C) -> %C == Config %Deploy the test application - + %Deploy the rel key {200, _} = setup_rels(C, setup), @@ -324,7 +315,7 @@ app_deploy(C) -> %C == Config ExpectedBoundConfg = maps:from_list([ {<<"services_calls">> , [<<"666.666.666.666:13">>]}, - {<<"streams_produces">>, [<<"666.666.666.666:13">>]}, + {<<"streams_produces">>, [<<"666.666.666.666:13">>]}, {<<"donotresolveme">> , <<"donotabsolveme">>} ]), @@ -346,13 +337,13 @@ app_deploy(C) -> %C == Config %assert the current appliccation list does not contain our test app {200,RB0} = httpabs:get(?XER, ?SC([?PLG(broker_url, C), "/application"])), true = lists:all(fun(X) -> X /= ?PLG(appname, C) end, jiffy:decode(RB0)), - + %deploy the app Body = valid_deploy_body(C), {200, RB} = httpabs:put(?XER, ?PLG(broker_app_url, C), "application/json", jiffy:encode(Body)), - + %The CDAP APIs return the config as a JSON dumped to a string, so we need to get that back into a real JSON to have key-order-independent equality testing - Fix = fun(X) -> + Fix = fun(X) -> RBMap = jiffy:decode(X, [return_maps]), maps:update(<<"bound_config">>, jiffy:decode(maps:get(<<"bound_config">>, RBMap), [return_maps]), RBMap) end, @@ -370,10 +361,10 @@ app_deploy(C) -> %C == Config %make sure it is in CDAP {200, _} = httpabs:get(?XER, ?PLG(app_url, C)), - + %check metrics {200, _} = httpabs:get(?XER, ?SC([?PLG(broker_app_url, C), "/metrics"])), - + %check healthcheck {200, _} = httpabs:get(?XER,?SC([?PLG(broker_app_url, C), "/healthcheck"])), @@ -383,7 +374,7 @@ app_deploy(C) -> %C == Config %check that the UNbound config is correct true = {200, util:ejson_to_map(?PLG(init_config, C))} == get_config_consul(C), - + %check that the preferences in Consul is correct InitPrefMap = util:ejson_to_map(?PLG(init_preferences, C)), true = {200, InitPrefMap} == get_preferences_consul(C), @@ -402,7 +393,7 @@ app_deploy(C) -> %C == Config true = ExpectedBoundConfg == CDAPConfig, %try to put the same app again and assert you get a 400 - {400,"State: Bad Request. Return Body: Put recieved on /application/:appname but appname is already registered. Call /application/:appname/reconfigure if trying to reconfigure or delete first"} = + {400,"State: Bad Request. Return Body: Put recieved on /application/:appname but appname is already registered. Call /application/:appname/reconfigure if trying to reconfigure or delete first"} = httpabs:put(?XER, ?PLG(broker_app_url, C), "application/json", jiffy:encode(Body)). hydrator_deploy(C) -> @@ -426,7 +417,7 @@ hydrator_deploy(C) -> %assert the current appliccation list does not contain our test app {200,RB0} = httpabs:get(?XER, ?SC([?PLG(broker_url, C), "/application"])), true = lists:all(fun(X) -> X /= ?PLG(hydrator_appname, C) end, jiffy:decode(RB0)), - + %try the deploy {200, RB1} = httpabs:put(?XER, ?PLG(broker_hydrator_app_url, C), "application/json", jiffy:encode(Body)), true = jiffy:decode(RB1, [return_maps]) == Expected, @@ -437,17 +428,17 @@ hydrator_deploy(C) -> %make sure it is in CDAP {200, _} = httpabs:get(?XER, ?PLG(hydrator_app_url, C)), - + %assert the current application list now includes our new app {200, RB3} = httpabs:get(?XER, ?SC([?PLG(broker_url, C), "/application"])), true = lists:any(fun(X) -> X == ?PLG(hydrator_appname, C) end, jiffy:decode(RB3)), - + %check healthcheck {200, _} = httpabs:get(?XER,?SC([?PLG(broker_hydrator_app_url, C), "/healthcheck"])), - + %check metrics {200, _} = httpabs:get(?XER,?SC([?PLG(broker_hydrator_app_url, C), "/metrics"])), - + %make sure that the service is registered. TODO! Could get more fancy by manually checking a healthcheck {200, RBHC} = httpabs:get(?XER,?PLG(consul_hydrator_app_url, C)), true = jiffy:decode(RBHC) /= [] @@ -486,7 +477,7 @@ hydrator_wdeps_deploy(C) -> %try the deploy {200, RB1} = httpabs:put(?XER, ?PLG(broker_hydrator_wdeps_app_url, C), "application/json", jiffy:encode(Body)), true = jiffy:decode(RB1, [return_maps]) == Expected, - + %make sure properties are loaded, test artifact {200, _} = httpabs:get(?XER,?SC([?PLG(cdap_ns_url, C), "/artifacts/", ?PLG(hydrator_wdeps_artname, C), "/versions/", ?PLG(hydrator_wdeps_artver, C), "/properties"])), @@ -496,22 +487,22 @@ hydrator_wdeps_deploy(C) -> %make sure it is in CDAP {200, _} = httpabs:get(?XER,?PLG(hydrator_wdeps_app_url, C)), - + %assert the current application list now includes our new app {200, RB3} = httpabs:get(?XER,?SC([?PLG(broker_url, C), "/application"])), true = lists:any(fun(X) -> X == ?PLG(hydrator_wdeps_appname, C) end, jiffy:decode(RB3)), - + %check healthcheck {200, _} = httpabs:get(?XER,?SC([?PLG(broker_hydrator_wdeps_app_url, C), "/healthcheck"])), - + %check metrics {200, _} = httpabs:get(?XER,?SC([?PLG(broker_hydrator_wdeps_app_url, C), "/metrics"])), - + %make sure that the service is registered. TODO! Could get more fancy by manually checking a healthcheck {200, RBHC} = httpabs:get(?XER,?PLG(consul_hydrator_wdeps_app_url, C)), true = jiffy:decode(RBHC) /= [] . - + hydrator_test(C) -> %test te app by injecting some data into the stream and getting it out %Sleeping since HTTP services may still be booting up: see https://issues.cask.co/browse/CDAP-812 @@ -554,7 +545,7 @@ app_reconfigure(C) -> true = {200, ReconfigMap} == get_config_consul(C), %test new config right in cdap true = {200, ReconfigMap} == get_config_cdap(C), - + %Test preferences reconfiguration %check that the preferences in Consul is correct InitMap = util:ejson_to_map(?PLG(init_preferences, C)), @@ -594,8 +585,8 @@ app_reconfigure(C) -> true = {200, ExpectedNewPreferencesBoth} == get_preferences_cdap(C), true = {200, ExpectedNewConfigBoth} == get_config_consul(C), true = {200, ExpectedNewConfigBoth} == get_config_cdap(C), - - %try to give it a smart where there are no overlaps + + %try to give it a smart where there are no overlaps SmartReconfigNone = {[{<<"EMPTY">>, <<"LIKE YOUR SOUL">>}]}, {400, _} = httpabs:put(?XER, ?SC([?PLG(broker_app_url, C), "/reconfigure"]), "application/json", jiffy:encode({[{<<"reconfiguration_type">>, <<"program-flowlet-smart">>},{<<"config">>, SmartReconfigNone}]})), true = {200, ExpectedNewPreferencesBoth} == get_preferences_consul(C), @@ -609,18 +600,18 @@ app_botch_flows(C) -> {200, _} = httpabs:get(?XER,?SC([?PLG(broker_app_url, C), "/healthcheck"])), %purposely shut down a flow "manually" to test that undeploy works with a "partially deployed" app - {200, []} = cdap_interface:exec_programs(?XER, ?PLG(appname, C), ?PLG(namespace, C), ?PLG(cdap_url, C), + {200, []} = cdap_interface:exec_programs(?XER, ?PLG(appname, C), ?PLG(namespace, C), ?PLG(cdap_url, C), [#program{type = <<"flows">>, id = <<"WhoFlow">>}, #program{type = <<"services">>, id = <<"Greeting">>}], "stop"), %make sure healthcheck now fails {400, _} = httpabs:get(?XER,?SC([?PLG(broker_app_url, C), "/healthcheck"])) . app_botch_delete(C) -> - %purposely shut down flows and then delete the app from the CDAP api to test undeploy works with a [gone] app - {200, []} = cdap_interface:exec_programs(?XER, ?PLG(appname, C), ?PLG(namespace, C), ?PLG(cdap_url, C), + %purposely shut down flows and then delete the app from the CDAP api to test undeploy works with a [gone] app + {200, []} = cdap_interface:exec_programs(?XER, ?PLG(appname, C), ?PLG(namespace, C), ?PLG(cdap_url, C), [#program{type = <<"flows">>, id = <<"WhoFlow">>}, #program{type = <<"services">>, id = <<"Greeting">>}], "stop"), {200, []} = cdap_interface:delete_app(?XER, ?PLG(appname, C), ?PLG(namespace, C), ?PLG(cdap_url, C)), - + %make sure healthcheck now fails {400, _} = httpabs:get(?XER,?SC([?PLG(broker_app_url, C), "/healthcheck"])) . @@ -636,7 +627,7 @@ app_teardown(C) -> %teardown the test application {200, []} = httpabs:delete(?XER, ?PLG(broker_app_url, C)), - + %make sure the broker deleted the config from Consul {404, _} = get_config_consul(C), @@ -645,17 +636,17 @@ app_teardown(C) -> %make sure the broker app url no longer exists {404, _ } = httpabs:get(?XER,?PLG(broker_app_url, C)), - + %teardown the testing rels {404, _} = setup_rels(C, teardown), - + %teardown the fake service and make sure it is gone {200, Srv} = setup_fake_testing_service(C, teardown), true = Srv == "[]", - + %cdap app gone {404,"State: Not Found. Return Body: 'application:testns.hwtest.-SNAPSHOT' was not found."} = httpabs:get(?XER,?PLG(app_url, C)), - + %make sure that the service is not registered. TODO! Could get more fancy by manually checking a healthcheck {200, RBHC} = httpabs:get(?XER,?PLG(consul_app_url, C)), true = jiffy:decode(RBHC) == []. @@ -727,9 +718,9 @@ hydrator_wdeps_teardown(C) -> test_failures(C) -> %test things that should fail %delete a non-existent app - {404, "State: Not Found. Return Body: Tried to delete an application that was not registered"} = + {404, "State: Not Found. Return Body: Tried to delete an application that was not registered"} = httpabs:delete(?XER, ?SC([?PLG(broker_app_url, C), "MYFRIENDOFMISERY"])), - + %malformed Broker put URL = ?SC([?PLG(broker_app_url, C), "FAILURETEST"]), Body = {[ @@ -747,7 +738,7 @@ test_failures(C) -> {<<"artifact_version">>, ?PLG(art_ver, C)}, {<<"app_config">>, ?PLG(init_config, C)}, {<<"app_preferences">>, ?PLG(init_preferences, C)}, - {<<"services">>, [{[{<<"service_name">>, <<"Greeting">>}, + {<<"services">>, [{[{<<"service_name">>, <<"Greeting">>}, {<<"service_endpoint">>, <<"greet">>}, {<<"endpoint_method">>, <<"GET">>}]}]}, {<<"programs">>, [ @@ -762,7 +753,7 @@ test_failures(C) -> {404,_} = httpabs:put(?XER, URL, "application/json", jiffy:encode(Body2)), %make sure the rollback happened {200, "[]"} = httpabs:get(?XER,?SC([?PLG(broker_url, C), "/application"])), - + %try to deploy with a bad URL where bad means nonexistent (504) Body3 = {[ {<<"cdap_application_type">>, <<"program-flowlet">>}, @@ -796,20 +787,6 @@ test_failures(C) -> {400,"State: Bad Request. Return Body: ERROR: The following URL is malformed: THIS IS NOT EVEN A URL WHAT ARE YOU DOING TO ME"} = httpabs:put(?XER, URL, "application/json", jiffy:encode(Body4)) . -invalid_reconfigure(C) -> - %test reconfiguring an app that does not exist despite put body being correct - {404,"State: Not Found. Return Body: Reconfigure recieved but the app is not registered"} = httpabs:put(?XER, ?SC([?PLG(broker_app_url, C), "THE_VOID", "/reconfigure"]), "application/json", jiffy:encode({[{<<"reconfiguration_type">>, <<"program-flowlet-app-config>">>}, {<<"config">>, {[{<<"foo">>, <<"bar">>}]}}]})), - - %test reconfiguring with an invalid PUT body (missing "reconfiguration_type") - {400,"State: Bad Request. Return Body: Invalid PUT Reconfigure Body: key 'reconfiguration_type' is missing"} = httpabs:put(?XER, ?SC([?PLG(broker_app_url,C), "/reconfigure"]), "application/json", jiffy:encode({[{<<"config">>, <<"bar">>}]})), - - %test reconfiguring with an invalid PUT body (missing app_config) - {400,"State: Bad Request. Return Body: Invalid PUT Reconfigure Body: key 'config' is missing"} = httpabs:put(?XER, ?SC([?PLG(broker_app_url,C), "/reconfigure"]), "application/json", jiffy:encode({[{<<"reconfiguration_type">>, <<"program-flowlet-app-config">>}, {<<"foo">>, <<"bar">>}]})), - - %test reconfiguring an invalid (unimplemented) type - {501, "State: Not Implemented. Return Body: This type (EMPTINESS) of reconfiguration is not implemented"} = httpabs:put(?XER, ?SC([?PLG(broker_app_url,C), "/reconfigure"]), "application/json", jiffy:encode({[{<<"config">>, <<"bar">>}, {<<"reconfiguration_type">>, <<"EMPTINESS">>}]})) - . - delete_all(C) -> %test invalid key Body1 = jiffy:encode({[{<<"ids">>, [<<"hwtest">>]}]}), @@ -823,4 +800,4 @@ delete_all(C) -> %teardown the fake service and make sure it is gone {200, Srv} = setup_fake_testing_service(C, teardown), true = Srv == "[]". - + |