From a82619114f3604d4f2695d6aef35976bfb33849a Mon Sep 17 00:00:00 2001 From: Roman Hargrave Date: Sun, 28 Aug 2022 19:30:00 -0700 Subject: [PATCH 01/10] feat: implement XEP-0363 with S3CS API --- include/aws.hrl | 28 +++ src/aws_util.erl | 255 ++++++++++++++++++++++++ src/mod_s3_upload.erl | 443 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 726 insertions(+) create mode 100644 include/aws.hrl create mode 100644 src/aws_util.erl create mode 100644 src/mod_s3_upload.erl diff --git a/include/aws.hrl b/include/aws.hrl new file mode 100644 index 000000000..d42f7a07a --- /dev/null +++ b/include/aws.hrl @@ -0,0 +1,28 @@ +%%%---------------------------------------------------------------------- +%%% File : s3_util.erl +%%% Usage : S3 URL Generation and Signing +%%% Author : Roman Hargrave +%%% Purpose : Signing AWS Requests. Intended for S3-CS use. +%%% Created : 24 Aug 2022 by Roman Hargrave +%%% +%%% +%%% This program is free software; you can redistribute it and/or +%%% modify it under the terms of the GNU General Public License as +%%% published by the Free Software Foundation; either version 2 of the +%%% License, or (at your option) any later version. +%%% +%%% This program is distributed in the hope that it will be useful, +%%% but WITHOUT ANY WARRANTY; without even the implied warranty of +%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +%%% General Public License for more details. +%%% +%%% You should have received a copy of the GNU General Public License along +%%% with this program; if not, write to the Free Software Foundation, Inc., +%%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +%%%---------------------------------------------------------------------- + +-record(aws_auth, {access_key_id :: binary(), + access_key :: binary(), + region :: binary()}). + +-define(AWS_SERVICE_S3, <<"s3">>). diff --git a/src/aws_util.erl b/src/aws_util.erl new file mode 100644 index 000000000..b6a1a87d8 --- /dev/null +++ b/src/aws_util.erl @@ -0,0 +1,255 @@ +%%%---------------------------------------------------------------------- +%%% File : s3_util.erl +%%% Usage : S3 URL Generation and Signing +%%% Author : Roman Hargrave +%%% Purpose : Signing AWS Requests. Intended for S3-CS use. +%%% Created : 24 Aug 2022 by Roman Hargrave +%%% +%%% +%%% This program is free software; you can redistribute it and/or +%%% modify it under the terms of the GNU General Public License as +%%% published by the Free Software Foundation; either version 2 of the +%%% License, or (at your option) any later version. +%%% +%%% This program is distributed in the hope that it will be useful, +%%% but WITHOUT ANY WARRANTY; without even the implied warranty of +%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +%%% General Public License for more details. +%%% +%%% You should have received a copy of the GNU General Public License along +%%% with this program; if not, write to the Free Software Foundation, Inc., +%%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +%%%---------------------------------------------------------------------- + +-module(aws_util). +-author("roman@hargrave.info"). + +-include("aws.hrl"). + +%% URL Signing. Documented at +%% https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html + +-type verb() :: get | put | post | delete. +-type headers() :: [{unicode:chardata(), unicode:chardata()}]. +-type query_list() :: [{unicode:chardata(), unicode:chardata() | true}]. +-type ttl() :: 1..604800. + +-define(AWS_SIGN_ALGO, <<"AWS4-HMAC-SHA256">>). + +-import(crypto, [mac/4]). +-import(uri_string, [compose_query/1, + dissect_query/1]). + +-export([signed_url/7]). + +%%------------------------------------------------------------------------ +%% API +%%------------------------------------------------------------------------ + +-spec signed_url( + Auth :: #aws_auth{}, + Verb :: verb(), + Service :: binary(), + Url :: binary(), + ExtraHeaders :: headers(), + Time :: calendar:datetime(), + TTL :: ttl() + ) -> + SignedUrl :: binary(). +% sign a URL given headers, a verb, authentication details, and a time +signed_url(Auth, Verb, Service, Url, ExtraHeaders, Time, TTL) -> + #{host := Host} = UnauthenticatedUriMap = uri_string:parse(Url), + Headers = [{<<"host">>, Host}|ExtraHeaders], + % insert authentication params. + QueryList = sorted_query_list(uri_query_list(UnauthenticatedUriMap) + ++ base_query_params(Auth, Time, Service, Headers, TTL)), + UriMap = UnauthenticatedUriMap#{query => compose_query(QueryList)}, + % generate and sign the message + StringToSign = string_to_sign(Auth, Time, Service, Verb, UriMap, Headers), + SigningKey = signing_key(Auth, Time, Service), + Signature = encode_hex(mac(hmac, sha256, SigningKey, StringToSign)), + % add signature to the query list and compose URI + SignedQueryString = compose_query([{<<"X-Amz-Signature">>, Signature}|QueryList]), + uri_string:recompose(UriMap#{query => SignedQueryString}). + +%%------------------------------------------------------------------------ +%% Internal +%%------------------------------------------------------------------------ + +-spec sorted_query_list( + query_list() + ) -> + query_list(). +% sort a query paramater list by parameter name, ascending +sorted_query_list(QueryList) -> + lists:sort(fun ({L, _}, {R, _}) -> L =< R end, QueryList). + +-spec uri_query_list( + uri_string:uri_map() + ) -> + query_list(). +% extract a query list from a uri_map(). +uri_query_list(#{query := QueryString}) -> + dissect_query(QueryString); +uri_query_list(_) -> + []. + +-spec verb( + verb() + ) -> + binary(). +% convert a verb atom to a binary list +verb(get) -> + <<"GET">>; +verb(put) -> + <<"PUT">>; +verb(post) -> + <<"POST">>; +verb(delete) -> + <<"DELETE">>. + +-spec encode_hex( + binary() + ) -> + binary(). +% lowercase binary:encode_hex +encode_hex(Data) -> + str:to_lower(binary:encode_hex(Data)). + +-spec iso8601_timestamp_utc( + calendar:datetime() + ) -> + binary(). +% Generate an ISO8601-like YmdTHMSZ timestamp for X-Amz-Date. Only +% produces UTC ('Z') timestamps. No separators. +iso8601_timestamp_utc({{Y, Mo, D}, {H, M, S}}) -> + str:format("~B~2..0B~2..0BT~2..0B~2..0B~2..0BZ", + [Y, Mo, D, + H, M, S]). + +-spec iso8601_date( + calendar:datetime() + ) -> + binary(). +% ISO8601 formatted date, no separators. +iso8601_date({{Y, M, D}, _}) -> + str:format("~B~2..0B~2..0B", [Y, M, D]). + +-spec scope( + #aws_auth{}, + calendar:datetime(), + binary() + ) -> + binary(). +% Generate the request scope used in the credential field and signature message +scope(#aws_auth{region = Region}, + Time, + Service) -> + str:format("~ts/~ts/~ts/aws4_request", + [iso8601_date(Time), + Region, + Service]). + +-spec credential( + #aws_auth{}, + calendar:datetime(), + binary() + ) -> + binary(). +% Generate the value used for X-Amz-Credential +credential(#aws_auth{access_key_id = KeyID} = Auth, + Time, + Service) -> + str:format("~ts/~ts", [KeyID, scope(Auth, Time, Service)]). + +-spec base_query_params( + #aws_auth{}, + calendar:datetime(), + binary(), + headers(), + ttl() + ) -> + [{unicode:chardata(), unicode:chardata()}]. +% Return the minimum required set of query parameters needed for +% authenticated signed requests. +base_query_params(Auth, Time, Service, Headers, TTL) -> + [{<<"X-Amz-Algorithm">>, ?AWS_SIGN_ALGO}, + {<<"X-Amz-Credential">>, credential(Auth, Time, Service)}, + {<<"X-Amz-Date">>, iso8601_timestamp_utc(Time)}, + {<<"X-Amz-Expires">>, erlang:integer_to_binary(TTL)}, + {<<"X-Amz-SignedHeaders">>, signed_headers(Headers)}]. + +-spec canonical_headers( + headers() + ) -> + unicode:chardata(). +% generate the header list for canonical_request +canonical_headers(Headers) -> + str:join(lists:map(fun ({Name, Value}) -> + str:format("~ts:~ts~n", [Name, Value]) + end, Headers), + <<>>). + +-spec signed_headers( + headers() + ) -> + unicode:chardata(). +% generate a semicolon-delimited list of headers, used to enumerate +% signed headers in the AWSv4 canonical request +signed_headers(SignedHeaders) -> + str:join(lists:map(fun ({Name, _}) -> + Name + end, SignedHeaders), + <<";">>). + +-spec canonical_request( + verb(), + uri_string:uri_map(), + headers() + ) -> + unicode:chardata(). +% Generate the canonical request used so compute the signature +canonical_request(Verb, + #{query := Query, + path := Path}, + Headers) -> + <<(verb(Verb))/binary, "\n", + Path/binary, "\n", + Query/binary, "\n", + (canonical_headers(Headers))/binary, "\n", + (signed_headers(Headers))/binary, "\n", + "UNSIGNED-PAYLOAD">>. + +-spec string_to_sign( + #aws_auth{}, + calendar:datetime(), + binary(), + verb(), + uri_string:uri_map(), + headers() + ) -> + unicode:chardata(). +% generate the "string to sign", as per AWS specs +string_to_sign(Auth, Time, Service, Verb, UriMap, Headers) -> + RequestHash = crypto:hash(sha256, canonical_request(Verb, UriMap, Headers)), + <>. + +-spec signing_key( + #aws_auth{}, + calendar:datetime(), + binary() + ) -> + binary(). +% generate the signing key used in the final HMAC-SHA256 round for +% request signing. +signing_key(#aws_auth{access_key = AccessKey, + region = Region}, + Time, + Service) -> + DateKey = mac(hmac, sha256, <<"AWS4", AccessKey/binary>>, iso8601_date(Time)), + DateRegionKey = mac(hmac, sha256, DateKey, Region), + DateRegionServiceKey = mac(hmac, sha256, DateRegionKey, Service), + mac(hmac, sha256, DateRegionServiceKey, <<"aws4_request">>). diff --git a/src/mod_s3_upload.erl b/src/mod_s3_upload.erl new file mode 100644 index 000000000..fb2ec72f8 --- /dev/null +++ b/src/mod_s3_upload.erl @@ -0,0 +1,443 @@ +%%%---------------------------------------------------------------------- +%%% File : mod_s3_upload.erl +%%% Author : Roman Hargrave +%%% Purpose : An XEP-0363 Implementation using S3-compatible storage +%%% Created : 24 Aug 2022 by Roman Hargrave +%%% +%%% +%%% This program is free software; you can redistribute it and/or +%%% modify it under the terms of the GNU General Public License as +%%% published by the Free Software Foundation; either version 2 of the +%%% License, or (at your option) any later version. +%%% +%%% This program is distributed in the hope that it will be useful, +%%% but WITHOUT ANY WARRANTY; without even the implied warranty of +%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +%%% General Public License for more details. +%%% +%%% You should have received a copy of the GNU General Public License along +%%% with this program; if not, write to the Free Software Foundation, Inc., +%%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +%%%---------------------------------------------------------------------- + +-module(mod_s3_upload). +-author('roman@hargrave.info'). + +-behaviour(gen_mod). +-behaviour(gen_server). + +-protocol({xep, 363, '1.1.0'}). + +-include("logger.hrl"). +-include("translate.hrl"). +-include("aws.hrl"). + +-include_lib("xmpp/include/xmpp.hrl"). + +% gen_mod callbacks +-export([start/2, + stop/1, + reload/3, + depends/2, + mod_opt_type/1, + mod_options/1, + mod_doc/0]). + +% gen_server callbacks +-export([init/1, + handle_info/2, + handle_call/3, + handle_cast/2]). + +-import(gen_mod, [get_opt/2]). + +%%----------------------------------------------------------------------- +%% gen_mod callbacks and related machinery +%%----------------------------------------------------------------------- + +-spec start( + binary(), + gen_mod:opts() + ) -> + {ok, pid()} | {error, term()}. +% +start(ServerHost, Opts) -> + gen_mod:start_child(?MODULE, ServerHost, Opts). + +-spec stop( + binary() + ) -> + any(). +% +stop(ServerHost) -> + gen_mod:stop_child(?MODULE, ServerHost). + +-spec reload( + binary(), + gen_mod:opts(), + gen_mod:opts() + ) -> + ok. +% +reload(ServerHost, NewOpts, _OldOpts) -> + ServerRef = gen_mod:get_module_proc(ServerHost, ?MODULE), + % cast a message to the server with the new options + gen_server:cast(ServerRef, {reload, + ServerHost, + build_service_params(ServerHost, NewOpts)}). + +%%------------------------------------------------------------------------ +%% Options +%%------------------------------------------------------------------------ + +mod_opt_type(access_key_id) -> + econf:binary(); +mod_opt_type(access_key_secret) -> + econf:binary(); +mod_opt_type(region) -> + econf:binary(); +mod_opt_type(bucket_url) -> + econf:url([http, https]); +mod_opt_type(max_size) -> + econf:pos_int(infinity); +mod_opt_type(set_public) -> + econf:bool(); +mod_opt_type(put_ttl) -> + econf:pos_int(infinity); +mod_opt_type(service_name) -> + econf:binary(); +mod_opt_type(hosts) -> + econf:hosts(); +mod_opt_type(access) -> + econf:acl(). + +mod_options(Host) -> + [{access_key_id, undefined}, + {access_key_secret, undefined}, + {region, undefined}, + {bucket_url, undefined}, + {max_size, 1073741824}, + {set_public, true}, + {put_ttl, 600}, + {service_name, <<"S3 Upload">>}, + {hosts, [<<"upload.", Host/binary>>]}, + {access, local}]. + +mod_doc() -> + #{desc => + [?T("This module implements XEP-0363 using an S3 bucket " + "instead of an internal web server. This simplifies " + "clustered deployments by removing the need to maintain " + "shared storage, and is in many cases less expensive " + "byte-for-byte than block storage. It is mutually " + "incompatible with mod_http_upload.")], + opts => + [{access_key_id, + #{value => ?T("AccessKeyId"), + desc => ?T("AWS Access Key ID.")}}, + {access_key_secret, + #{value => ?T("AccessKeySecret"), + desc => ?T("AWS Access Key Secret.")}}, + {region, + #{value => ?T("Region"), + desc => ?T("AWS Region")}}, + {bucket_url, + #{value => ?T("BucketUrl"), + desc => ?T("S3 Bucket URL.")}}, + {max_size, + #{value => ?T("MaxSize"), + desc => ?T("Maximum file size, in bytes. 0 is unlimited.")}}, + {set_public, + #{value => ?T("SetPublic"), + desc => ?T("Set x-amz-acl to public-read.")}}, + {put_ttl, + #{value => ?T("PutTtl"), + desc => ?T("How long the PUT URL will be valid for.")}}, + {service_name, + #{value => ?T("ServiceName"), + desc => ?T("Name given in discovery requests.")}}, + {hosts, % named for consistency with other modules + #{value => ?T("ServiceJids"), + desc => ?T("JIDs used when communicating with the service")}}, + {access, + #{value => ?T("UploadAccess"), + desc => ?T("Access rule for JIDs that may request new URLs")}}]}. + +depends(_Host, _Opts) -> + []. + +%%------------------------------------------------------------------------ +%% gen_server callbacks. +%%------------------------------------------------------------------------ + +-record(params, + {service_name :: binary(), % name given for the service in discovery. + service_jids :: [binary()], % stanzas destined for these JIDs will be routed to the service. + max_size :: integer() | infinity, % maximum upload size. sort of the honor system in this case. + bucket_url :: binary(), % S3 bucket URL or subdomain + set_public :: boolean(), % set the public-read ACL on the object? + ttl :: integer(), % TTL of the signed PUT URL + server_host :: binary(), % XMPP vhost the service belongs to + auth :: #aws_auth{}, + access :: atom()}). + +-spec init( + list() + ) -> + {ok, gen_mod:opts()}. +% +init([ServerHost, Opts]) -> + Params = build_service_params(ServerHost, Opts), + update_routes(ServerHost, [], Params#params.service_jids), + {ok, Params}. + +-spec handle_info( + _, + gen_mod:opts() + ) -> + {noreply, gen_mod:opts()}. +% receive non-standard (gen_server) messages +handle_info({route, #iq{lang = Lang} = Packet}, Opts) -> + try xmpp:decode_els(Packet) of + IQ -> + ejabberd_router:route(handle_iq(IQ, Opts)), + {noreply, Opts} + catch _:{xmpp_codec, Why} -> + Message = xmpp:io_format_error(Why), + Error = xmpp:err_bad_request(Message, Lang), + ejabbered_router:route_error(Packet, Error), + {noreply, Opts} + end; +handle_info(Request, Opts) -> + ?WARNING_MSG("Unexpected info: ~p", [Request]), + {noreply, Opts}. + +-spec handle_call( + _, + gen_server:from(), + gen_mod:opts() + ) -> + {_, gen_mod:opts()}. +% respond to $gen_call messages +handle_call(Request, Sender, Opts) -> + ?WARNING_MSG("Unexpected call from ~p: ~p", [Sender, Request]), + {noreply, Opts}. + +-spec handle_cast( + _, + gen_mod:opts() + ) -> + {_, gen_mod:opts()}. +% receive $gen_cast messages +handle_cast({reload, ServerHost, NewOpts}, OldOpts) -> + update_routes(ServerHost, + OldOpts#params.service_jids, + NewOpts#params.service_jids), + {ok, NewOpts}; +handle_cast(Request, Opts) -> + ?WARNING_MSG("Unexpected cast: ~p", [Request]), + {noreply, Opts}. + +%%------------------------------------------------------------------------ +%% Internal Stanza Processing +%%----------------------------------------------------------------------- + +-spec update_routes( + binary(), + [binary()], + [binary()] + ) -> + _. +% maintain routing rules for JIDs owned by this service. +update_routes(ServerHost, OldDomains, NewDomains) -> + lists:foreach(fun (Domain) -> + ejabberd_router:register_route(Domain, ServerHost) + end, NewDomains), + lists:foreach(fun ejabberd_router:unregister_route/1, OldDomains -- NewDomains). + + +-spec handle_iq( + iq(), + gen_mod:opts() + ) -> + iq(). +% Handle discovery requests. Produces a document such as depicted in +% XEP-0363 v1.1.0 Ex. 4. +handle_iq(#iq{type = get, + lang = Lang, + to = Host, + sub_els = [#disco_info{}]} = IQ, + #params{max_size = MaxSize, service_name = ServiceName}) -> + % collect additional discovery entries, if any. + Advice = ejabberd_hooks:run_fold(disco_info, Host, [], + [Host, ?MODULE, <<"">>, Lang]), + % if a maximum size was specified, append xdata with the limit + XData = case MaxSize of + infinity -> + Advice; + _ -> + [#xdata{type = result, + fields = http_upload:encode( + [{'max-file-size', MaxSize}], + ?NS_HTTP_UPLOAD_0, + Lang + )} + | Advice] + end, + % build disco iq + Query = #disco_info{identities = [#identity{category = <<"store">>, + type = <<"file">>, + name = translate:translate(Lang, ServiceName)}], + features = [?NS_HTTP_UPLOAD_0], + xdata = XData}, + xmpp:make_iq_result(IQ, Query); % this swaps parties for us +% handle slot request with FileSize > MaxSize +handle_iq(#iq{type = get, + from = From, + lang = Lang, + sub_els = [#upload_request_0{size = FileSize, + filename = FileName}]} = IQ, + #params{max_size = MaxSize}) when FileSize > MaxSize -> + ?WARNING_MSG("~ts tried to upload an oversize file (~ts, ~B bytes)", + [jid:encode(From), FileName, FileSize]), + ErrorMessage = {?T("File larger than ~B bytes"), [MaxSize]}, + Error = xmpp:err_not_acceptable(ErrorMessage, Lang), + Els = [#upload_file_too_large{'max-file-size' = MaxSize, + xmlns = ?NS_HTTP_UPLOAD_0} + | xmpp:get_els(Error)], + xmpp:make_error(IQ, xmpp:set_els(Error, Els)); +% Handle slot request +handle_iq(#iq{type = get, + from = Requester, + lang = Lang, + sub_els = [#upload_request_0{filename = FileName, + size = FileSize} = UploadRequest]} = IQ, + #params{server_host = ServerHost, + access = Access, + bucket_url = BucketURL, + ttl = TTL, + auth = Auth} = Params) -> + case acl:match_rule(ServerHost, Access, Requester) of + allow -> + ?INFO_MSG("Generating S3 Object URL Pair for ~ts to upload file ~ts (~B bytes)", + [jid:encode(Requester), FileName, FileSize]), + % generate a unique object ID and url based on settings + ObjectURL = object_url(BucketURL, FileName), + % attach configuration- and request-specific query params to the + % PUT url + UnsignedPutURL = put_url(UploadRequest, Params, ObjectURL), + % sign the PUT url + PutURL = aws_util:signed_url(Auth, put, ?AWS_SERVICE_S3, UnsignedPutURL, [], calendar:universal_time(), TTL), + xmpp:make_iq_result(IQ, #upload_slot_0{get = ObjectURL, + put = PutURL, + xmlns = ?NS_HTTP_UPLOAD_0}); + deny -> + ?INFO_MSG("Denied upload request from ~ts for file ~ts (~B bytes)", + [jid:encode(Requester), FileName, FileSize]), + xmpp:make_error(IQ, xmpp:err_forbidden(?T("Access denied"), Lang)) + end; +% handle unexpected IQ +handle_iq(IQ, _Params) -> + xmpp:make_error(IQ, xmpp:err_bad_request()). + +%%------------------------------------------------------------------------ +%% Internal Helpers +%%------------------------------------------------------------------------ + +-spec expanded_jids( + ServiceHost :: binary(), + JIDs :: [binary()] + ) -> + [binary()]. +% expand @HOST@ in JIDs +expanded_jids(ServerHost, JIDs) -> + lists:map(fun (JID) -> + misc:expand_keyword(<<"@HOST@">>, JID, ServerHost) + end, JIDs). + +-spec build_service_params( + binary(), + gen_mod:opts() + ) -> + #params{}. +% create a service params record from module config +build_service_params(ServerHost, Opts) -> + Auth = #aws_auth{access_key_id = get_opt(access_key_id, Opts), + access_key = get_opt(access_key_secret, Opts), + region = get_opt(region, Opts)}, + #params{service_name = get_opt(service_name, Opts), + service_jids = expanded_jids(ServerHost, get_opt(hosts, Opts)), + max_size = get_opt(max_size, Opts), + bucket_url = get_opt(bucket_url, Opts), + set_public = get_opt(set_public, Opts), + ttl = get_opt(put_ttl, Opts), + server_host = ServerHost, + auth = Auth, + access = get_opt(access, Opts)}. + +-spec url_service_parameters( + #params{} + ) -> + [{binary(), binary() | true}]. +% additional URL parameters from module config +url_service_parameters(#params{set_public = true}) -> + [{<<"X-Amz-Acl">>, <<"public-read">>}]; +url_service_parameters(_) -> + []. + +-spec upload_parameters( + #upload_request_0{}, + #params{} + ) -> + [{binary(), binary() | true}]. +% headers to be included with the PUT request +upload_parameters(#upload_request_0{size = FileSize, + 'content-type' = ContentType}, + ServiceParams) -> + [{<<"Content-Type">>, <>}, + {<<"Content-Length">>, erlang:integer_to_binary(FileSize)} + | url_service_parameters(ServiceParams)]. + +-spec put_url( + #upload_request_0{}, + #params{}, + binary() + ) -> + binary(). +% attach additional query parameters (to the PUT URL), specifically canned ACL. +put_url(UploadRequest, ServiceParams, URL) -> + UriMap = uri_string:parse(URL), + QueryList = case UriMap of + #{query := QueryString} -> + uri_string:dissect_query(QueryString); + _ -> + [] + end, + Params = upload_parameters(UploadRequest, ServiceParams), + WithOpts = uri_string:compose_query(Params ++ QueryList), + uri_string:recompose(UriMap#{query => WithOpts}). + +-spec object_url( + binary(), + binary() + ) -> + binary(). +% generate a unique random object URL for the given filename +object_url(BucketURL, FileName) -> + #{path := BasePath} = UriMap = uri_string:parse(BucketURL), + ObjectName = object_name(FileName), + uri_string:recompose(UriMap#{path => <>}). + +-spec object_name( + binary() + ) -> + binary(). +% generate a unique-in-time object name +object_name(FileName) -> + MD = crypto:hash_init(sha256), + MDFilename = crypto:hash_update(MD, FileName), + MDNodeName = crypto:hash_update(MDFilename, erlang:atom_to_binary(node())), + MDTime = crypto:hash_update(MDNodeName, <<(os:system_time())>>), + MDRand = crypto:hash_update(MDTime, crypto:strong_rand_bytes(256)), + Hash = crypto:hash_final(MDRand), + <<(binary:encode_hex(Hash))/binary, "-", FileName/binary>>. From aa12b7816e89ec8deb39c018c1f1836df5c48630 Mon Sep 17 00:00:00 2001 From: Roman Hargrave Date: Thu, 8 Sep 2022 13:07:54 -0700 Subject: [PATCH 02/10] style: give names in type specs, minor formatting in aws_util --- src/aws_util.erl | 98 ++++++++++++++++++++++++------------------------ 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/src/aws_util.erl b/src/aws_util.erl index b6a1a87d8..791e304ee 100644 --- a/src/aws_util.erl +++ b/src/aws_util.erl @@ -21,14 +21,14 @@ %%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. %%%---------------------------------------------------------------------- +%% URL Signing. Documented at +%% https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html + -module(aws_util). -author("roman@hargrave.info"). -include("aws.hrl"). -%% URL Signing. Documented at -%% https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html - -type verb() :: get | put | post | delete. -type headers() :: [{unicode:chardata(), unicode:chardata()}]. -type query_list() :: [{unicode:chardata(), unicode:chardata() | true}]. @@ -59,7 +59,7 @@ % sign a URL given headers, a verb, authentication details, and a time signed_url(Auth, Verb, Service, Url, ExtraHeaders, Time, TTL) -> #{host := Host} = UnauthenticatedUriMap = uri_string:parse(Url), - Headers = [{<<"host">>, Host}|ExtraHeaders], + Headers = [{<<"host">>, Host} | ExtraHeaders], % insert authentication params. QueryList = sorted_query_list(uri_query_list(UnauthenticatedUriMap) ++ base_query_params(Auth, Time, Service, Headers, TTL)), @@ -77,17 +77,17 @@ signed_url(Auth, Verb, Service, Url, ExtraHeaders, Time, TTL) -> %%------------------------------------------------------------------------ -spec sorted_query_list( - query_list() + QueryList :: query_list() ) -> - query_list(). + SortedQueryList :: query_list(). % sort a query paramater list by parameter name, ascending sorted_query_list(QueryList) -> lists:sort(fun ({L, _}, {R, _}) -> L =< R end, QueryList). -spec uri_query_list( - uri_string:uri_map() + UriMap :: uri_string:uri_map() ) -> - query_list(). + QueryList :: query_list(). % extract a query list from a uri_map(). uri_query_list(#{query := QueryString}) -> dissect_query(QueryString); @@ -95,7 +95,7 @@ uri_query_list(_) -> []. -spec verb( - verb() + Verb :: verb() ) -> binary(). % convert a verb atom to a binary list @@ -109,17 +109,17 @@ verb(delete) -> <<"DELETE">>. -spec encode_hex( - binary() + Data :: binary() ) -> - binary(). + EncodedData :: binary(). % lowercase binary:encode_hex encode_hex(Data) -> str:to_lower(binary:encode_hex(Data)). -spec iso8601_timestamp_utc( - calendar:datetime() + DateTime :: calendar:datetime() ) -> - binary(). + Timestamp :: binary(). % Generate an ISO8601-like YmdTHMSZ timestamp for X-Amz-Date. Only % produces UTC ('Z') timestamps. No separators. iso8601_timestamp_utc({{Y, Mo, D}, {H, M, S}}) -> @@ -128,19 +128,19 @@ iso8601_timestamp_utc({{Y, Mo, D}, {H, M, S}}) -> H, M, S]). -spec iso8601_date( - calendar:datetime() + DateTime :: calendar:datetime() ) -> - binary(). + DateStr :: binary(). % ISO8601 formatted date, no separators. iso8601_date({{Y, M, D}, _}) -> str:format("~B~2..0B~2..0B", [Y, M, D]). -spec scope( - #aws_auth{}, - calendar:datetime(), - binary() + Auth :: #aws_auth{}, + Time :: calendar:datetime(), + Service :: binary() ) -> - binary(). + Scope :: binary(). % Generate the request scope used in the credential field and signature message scope(#aws_auth{region = Region}, Time, @@ -151,11 +151,11 @@ scope(#aws_auth{region = Region}, Service]). -spec credential( - #aws_auth{}, - calendar:datetime(), - binary() + Auth :: #aws_auth{}, + Time :: calendar:datetime(), + Service :: binary() ) -> - binary(). + Auth :: binary(). % Generate the value used for X-Amz-Credential credential(#aws_auth{access_key_id = KeyID} = Auth, Time, @@ -163,13 +163,13 @@ credential(#aws_auth{access_key_id = KeyID} = Auth, str:format("~ts/~ts", [KeyID, scope(Auth, Time, Service)]). -spec base_query_params( - #aws_auth{}, - calendar:datetime(), - binary(), - headers(), - ttl() + Auth :: #aws_auth{}, + Time :: calendar:datetime(), + Service :: binary(), + Headers :: headers(), + TTL :: ttl() ) -> - [{unicode:chardata(), unicode:chardata()}]. + BaseQueryParams :: [{unicode:chardata(), unicode:chardata()}]. % Return the minimum required set of query parameters needed for % authenticated signed requests. base_query_params(Auth, Time, Service, Headers, TTL) -> @@ -180,9 +180,9 @@ base_query_params(Auth, Time, Service, Headers, TTL) -> {<<"X-Amz-SignedHeaders">>, signed_headers(Headers)}]. -spec canonical_headers( - headers() + Headers :: headers() ) -> - unicode:chardata(). + CanonicalHeaders :: unicode:chardata(). % generate the header list for canonical_request canonical_headers(Headers) -> str:join(lists:map(fun ({Name, Value}) -> @@ -191,9 +191,9 @@ canonical_headers(Headers) -> <<>>). -spec signed_headers( - headers() + SignedHeaders :: headers() ) -> - unicode:chardata(). + SignedHeaders :: unicode:chardata(). % generate a semicolon-delimited list of headers, used to enumerate % signed headers in the AWSv4 canonical request signed_headers(SignedHeaders) -> @@ -203,12 +203,12 @@ signed_headers(SignedHeaders) -> <<";">>). -spec canonical_request( - verb(), - uri_string:uri_map(), - headers() + Verb :: verb(), + UriMap :: uri_string:uri_map(), + Headers :: headers() ) -> - unicode:chardata(). -% Generate the canonical request used so compute the signature + CanonicalRequest :: unicode:chardata(). +% Generate the canonical request used to compute the signature canonical_request(Verb, #{query := Query, path := Path}, @@ -221,14 +221,14 @@ canonical_request(Verb, "UNSIGNED-PAYLOAD">>. -spec string_to_sign( - #aws_auth{}, - calendar:datetime(), - binary(), - verb(), - uri_string:uri_map(), - headers() + Auth :: #aws_auth{}, + Time :: calendar:datetime(), + Service :: binary(), + Verb :: verb(), + UriMap :: uri_string:uri_map(), + Headers :: headers() ) -> - unicode:chardata(). + StringToSign :: unicode:chardata(). % generate the "string to sign", as per AWS specs string_to_sign(Auth, Time, Service, Verb, UriMap, Headers) -> RequestHash = crypto:hash(sha256, canonical_request(Verb, UriMap, Headers)), @@ -238,11 +238,11 @@ string_to_sign(Auth, Time, Service, Verb, UriMap, Headers) -> (encode_hex(RequestHash))/binary>>. -spec signing_key( - #aws_auth{}, - calendar:datetime(), - binary() + Auth :: #aws_auth{}, + Time :: calendar:datetime(), + Service :: binary() ) -> - binary(). + SigningKey :: binary(). % generate the signing key used in the final HMAC-SHA256 round for % request signing. signing_key(#aws_auth{access_key = AccessKey, From 2e89a4040258becb5d1351fedbf6edcb07988955 Mon Sep 17 00:00:00 2001 From: Roman Hargrave Date: Thu, 8 Sep 2022 13:08:38 -0700 Subject: [PATCH 03/10] fix: typo in mod_s3_upload --- src/mod_s3_upload.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mod_s3_upload.erl b/src/mod_s3_upload.erl index fb2ec72f8..cf6a72fd7 100644 --- a/src/mod_s3_upload.erl +++ b/src/mod_s3_upload.erl @@ -205,7 +205,7 @@ handle_info({route, #iq{lang = Lang} = Packet}, Opts) -> catch _:{xmpp_codec, Why} -> Message = xmpp:io_format_error(Why), Error = xmpp:err_bad_request(Message, Lang), - ejabbered_router:route_error(Packet, Error), + ejabberd_router:route_error(Packet, Error), {noreply, Opts} end; handle_info(Request, Opts) -> From bf0984e031d9b8fd92073b238a52c2f1b345bdb6 Mon Sep 17 00:00:00 2001 From: Roman Hargrave Date: Thu, 8 Sep 2022 13:15:51 -0700 Subject: [PATCH 04/10] fix: type spec names, use `/` instead of `-` in object name The change from - to / was made because object storage doesn't place terrible importance upon `/` in names, while clients will treat all text following / as the file name. --- src/mod_s3_upload.erl | 114 ++++++++++++++++++++++++------------------ 1 file changed, 64 insertions(+), 50 deletions(-) diff --git a/src/mod_s3_upload.erl b/src/mod_s3_upload.erl index cf6a72fd7..848c13fd7 100644 --- a/src/mod_s3_upload.erl +++ b/src/mod_s3_upload.erl @@ -56,28 +56,28 @@ %%----------------------------------------------------------------------- -spec start( - binary(), - gen_mod:opts() + ServerHost :: binary(), + Opts :: gen_mod:opts() ) -> - {ok, pid()} | {error, term()}. + Result :: {ok, pid()} | {error, term()}. % start(ServerHost, Opts) -> gen_mod:start_child(?MODULE, ServerHost, Opts). -spec stop( - binary() + ServerHost :: binary() ) -> - any(). + Result :: any(). % stop(ServerHost) -> gen_mod:stop_child(?MODULE, ServerHost). -spec reload( - binary(), - gen_mod:opts(), - gen_mod:opts() + ServerHost :: binary(), + NewOpts :: gen_mod:opts(), + OldOpts :: gen_mod:opts() ) -> - ok. + Result :: ok. % reload(ServerHost, NewOpts, _OldOpts) -> ServerRef = gen_mod:get_module_proc(ServerHost, ?MODULE), @@ -90,6 +90,11 @@ reload(ServerHost, NewOpts, _OldOpts) -> %% Options %%------------------------------------------------------------------------ +-spec mod_opt_type( + OptionName :: atom() + ) -> + OptionType :: econf:validator(). +% mod_opt_type(access_key_id) -> econf:binary(); mod_opt_type(access_key_secret) -> @@ -111,6 +116,11 @@ mod_opt_type(hosts) -> mod_opt_type(access) -> econf:acl(). +-spec mod_options( + Host :: binary() + ) -> + Options :: [{atom, any()}]. +% mod_options(Host) -> [{access_key_id, undefined}, {access_key_secret, undefined}, @@ -123,6 +133,10 @@ mod_options(Host) -> {hosts, [<<"upload.", Host/binary>>]}, {access, local}]. +-spec mod_doc() -> + Doc :: #{desc => binary() | [binary()], + opts => [{atom(), #{value := binary(), desc := binary()}}]}. +% mod_doc() -> #{desc => [?T("This module implements XEP-0363 using an S3 bucket " @@ -182,9 +196,9 @@ depends(_Host, _Opts) -> access :: atom()}). -spec init( - list() + Params :: list() ) -> - {ok, gen_mod:opts()}. + Result :: {ok, gen_mod:opts()}. % init([ServerHost, Opts]) -> Params = build_service_params(ServerHost, Opts), @@ -192,10 +206,10 @@ init([ServerHost, Opts]) -> {ok, Params}. -spec handle_info( - _, - gen_mod:opts() + Message :: any(), + State :: gen_mod:opts() ) -> - {noreply, gen_mod:opts()}. + Result :: {noreply, gen_mod:opts()}. % receive non-standard (gen_server) messages handle_info({route, #iq{lang = Lang} = Packet}, Opts) -> try xmpp:decode_els(Packet) of @@ -213,21 +227,21 @@ handle_info(Request, Opts) -> {noreply, Opts}. -spec handle_call( - _, - gen_server:from(), - gen_mod:opts() + Request:: any(), + Sender :: gen_server:from(), + State :: gen_mod:opts() ) -> - {_, gen_mod:opts()}. + Result :: {_, gen_mod:opts()}. % respond to $gen_call messages handle_call(Request, Sender, Opts) -> ?WARNING_MSG("Unexpected call from ~p: ~p", [Sender, Request]), {noreply, Opts}. -spec handle_cast( - _, - gen_mod:opts() + Request :: any(), + State :: gen_mod:opts() ) -> - {_, gen_mod:opts()}. + Result :: {_, gen_mod:opts()}. % receive $gen_cast messages handle_cast({reload, ServerHost, NewOpts}, OldOpts) -> update_routes(ServerHost, @@ -243,24 +257,24 @@ handle_cast(Request, Opts) -> %%----------------------------------------------------------------------- -spec update_routes( - binary(), - [binary()], - [binary()] + ServerHost :: binary(), + OldJIDs :: [binary()], + NewJIDs :: [binary()] ) -> - _. + Result :: _. % maintain routing rules for JIDs owned by this service. -update_routes(ServerHost, OldDomains, NewDomains) -> +update_routes(ServerHost, OldJIDs, NewJIDs) -> lists:foreach(fun (Domain) -> ejabberd_router:register_route(Domain, ServerHost) - end, NewDomains), - lists:foreach(fun ejabberd_router:unregister_route/1, OldDomains -- NewDomains). + end, NewJIDs), + lists:foreach(fun ejabberd_router:unregister_route/1, OldJIDs -- NewJIDs). -spec handle_iq( - iq(), - gen_mod:opts() + IQ :: iq(), + Params :: gen_mod:opts() ) -> - iq(). + Response :: iq(). % Handle discovery requests. Produces a document such as depicted in % XEP-0363 v1.1.0 Ex. 4. handle_iq(#iq{type = get, @@ -348,7 +362,7 @@ handle_iq(IQ, _Params) -> ServiceHost :: binary(), JIDs :: [binary()] ) -> - [binary()]. + ExpandedJIDs :: [binary()]. % expand @HOST@ in JIDs expanded_jids(ServerHost, JIDs) -> lists:map(fun (JID) -> @@ -356,10 +370,10 @@ expanded_jids(ServerHost, JIDs) -> end, JIDs). -spec build_service_params( - binary(), - gen_mod:opts() + ServerHost :: binary(), + Opts :: gen_mod:opts() ) -> - #params{}. + Params :: #params{}. % create a service params record from module config build_service_params(ServerHost, Opts) -> Auth = #aws_auth{access_key_id = get_opt(access_key_id, Opts), @@ -376,9 +390,9 @@ build_service_params(ServerHost, Opts) -> access = get_opt(access, Opts)}. -spec url_service_parameters( - #params{} + Params :: #params{} ) -> - [{binary(), binary() | true}]. + ServiceParameters :: [{binary(), binary() | true}]. % additional URL parameters from module config url_service_parameters(#params{set_public = true}) -> [{<<"X-Amz-Acl">>, <<"public-read">>}]; @@ -386,10 +400,10 @@ url_service_parameters(_) -> []. -spec upload_parameters( - #upload_request_0{}, - #params{} + UploadRequest :: #upload_request_0{}, + Params :: #params{} ) -> - [{binary(), binary() | true}]. + UploadParameters :: [{binary(), binary() | true}]. % headers to be included with the PUT request upload_parameters(#upload_request_0{size = FileSize, 'content-type' = ContentType}, @@ -399,11 +413,11 @@ upload_parameters(#upload_request_0{size = FileSize, | url_service_parameters(ServiceParams)]. -spec put_url( - #upload_request_0{}, - #params{}, - binary() + UploadRequest :: #upload_request_0{}, + Params :: #params{}, + URL :: binary() ) -> - binary(). + PutURL :: binary(). % attach additional query parameters (to the PUT URL), specifically canned ACL. put_url(UploadRequest, ServiceParams, URL) -> UriMap = uri_string:parse(URL), @@ -418,10 +432,10 @@ put_url(UploadRequest, ServiceParams, URL) -> uri_string:recompose(UriMap#{query => WithOpts}). -spec object_url( - binary(), - binary() + BucketURL :: binary(), + FileName :: binary() ) -> - binary(). + ObjectURL :: binary(). % generate a unique random object URL for the given filename object_url(BucketURL, FileName) -> #{path := BasePath} = UriMap = uri_string:parse(BucketURL), @@ -429,9 +443,9 @@ object_url(BucketURL, FileName) -> uri_string:recompose(UriMap#{path => <>}). -spec object_name( - binary() + FileName :: binary() ) -> - binary(). + ObjectName :: binary(). % generate a unique-in-time object name object_name(FileName) -> MD = crypto:hash_init(sha256), @@ -440,4 +454,4 @@ object_name(FileName) -> MDTime = crypto:hash_update(MDNodeName, <<(os:system_time())>>), MDRand = crypto:hash_update(MDTime, crypto:strong_rand_bytes(256)), Hash = crypto:hash_final(MDRand), - <<(binary:encode_hex(Hash))/binary, "-", FileName/binary>>. + <<(binary:encode_hex(Hash))/binary, "/", FileName/binary>>. From 5168c145fffac5c28423fdca4a4b8de3a654f6fd Mon Sep 17 00:00:00 2001 From: Roman Hargrave Date: Thu, 8 Sep 2022 21:24:16 -0700 Subject: [PATCH 05/10] style: indentation in aws.hrl --- include/aws.hrl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/aws.hrl b/include/aws.hrl index d42f7a07a..42940639a 100644 --- a/include/aws.hrl +++ b/include/aws.hrl @@ -22,7 +22,7 @@ %%%---------------------------------------------------------------------- -record(aws_auth, {access_key_id :: binary(), - access_key :: binary(), - region :: binary()}). + access_key :: binary(), + region :: binary()}). -define(AWS_SERVICE_S3, <<"s3">>). From ffd2607ada7208a686d1d7e0dd88a13bf20be2fd Mon Sep 17 00:00:00 2001 From: Roman Hargrave Date: Thu, 8 Sep 2022 21:46:44 -0700 Subject: [PATCH 06/10] fix(mod_s3_upload): type specs, contract adherance --- src/mod_s3_upload.erl | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/mod_s3_upload.erl b/src/mod_s3_upload.erl index 848c13fd7..545ccfb90 100644 --- a/src/mod_s3_upload.erl +++ b/src/mod_s3_upload.erl @@ -119,7 +119,7 @@ mod_opt_type(access) -> -spec mod_options( Host :: binary() ) -> - Options :: [{atom, any()}]. + Options :: [{atom(), term()} | atom()]. % mod_options(Host) -> [{access_key_id, undefined}, @@ -198,7 +198,7 @@ depends(_Host, _Opts) -> -spec init( Params :: list() ) -> - Result :: {ok, gen_mod:opts()}. + Result :: {ok, #params{}}. % init([ServerHost, Opts]) -> Params = build_service_params(ServerHost, Opts), @@ -207,9 +207,9 @@ init([ServerHost, Opts]) -> -spec handle_info( Message :: any(), - State :: gen_mod:opts() + State :: #params{} ) -> - Result :: {noreply, gen_mod:opts()}. + Result :: {noreply, #params{}}. % receive non-standard (gen_server) messages handle_info({route, #iq{lang = Lang} = Packet}, Opts) -> try xmpp:decode_els(Packet) of @@ -229,9 +229,9 @@ handle_info(Request, Opts) -> -spec handle_call( Request:: any(), Sender :: gen_server:from(), - State :: gen_mod:opts() + State :: #params{} ) -> - Result :: {_, gen_mod:opts()}. + Result :: {noreply, #params{}}. % respond to $gen_call messages handle_call(Request, Sender, Opts) -> ?WARNING_MSG("Unexpected call from ~p: ~p", [Sender, Request]), @@ -239,15 +239,15 @@ handle_call(Request, Sender, Opts) -> -spec handle_cast( Request :: any(), - State :: gen_mod:opts() + State :: #params{} ) -> - Result :: {_, gen_mod:opts()}. + Result :: {noreply, #params{}}. % receive $gen_cast messages handle_cast({reload, ServerHost, NewOpts}, OldOpts) -> update_routes(ServerHost, OldOpts#params.service_jids, NewOpts#params.service_jids), - {ok, NewOpts}; + {noreply, NewOpts}; handle_cast(Request, Opts) -> ?WARNING_MSG("Unexpected cast: ~p", [Request]), {noreply, Opts}. @@ -279,9 +279,10 @@ update_routes(ServerHost, OldJIDs, NewJIDs) -> % XEP-0363 v1.1.0 Ex. 4. handle_iq(#iq{type = get, lang = Lang, - to = Host, + to = HostJID, sub_els = [#disco_info{}]} = IQ, #params{max_size = MaxSize, service_name = ServiceName}) -> + Host = jid:encode(HostJID), % collect additional discovery entries, if any. Advice = ejabberd_hooks:run_fold(disco_info, Host, [], [Host, ?MODULE, <<"">>, Lang]), From 5cf7add61a8b135b1e30cc53cf8615c8fe0d97c9 Mon Sep 17 00:00:00 2001 From: Roman Hargrave Date: Fri, 9 Sep 2022 22:59:00 -0700 Subject: [PATCH 07/10] doc: fix mod_s3_upload comment --- src/aws_util.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aws_util.erl b/src/aws_util.erl index 791e304ee..31c0388ba 100644 --- a/src/aws_util.erl +++ b/src/aws_util.erl @@ -1,6 +1,6 @@ %%%---------------------------------------------------------------------- -%%% File : s3_util.erl -%%% Usage : S3 URL Generation and Signing +%%% File : aws_util.erl +%%% Usage : AWS URL Signing %%% Author : Roman Hargrave %%% Purpose : Signing AWS Requests. Intended for S3-CS use. %%% Created : 24 Aug 2022 by Roman Hargrave From ce2a4a7c1f63d590b416d9bcabcc5bbedd596d18 Mon Sep 17 00:00:00 2001 From: Roman Hargrave Date: Fri, 9 Sep 2022 22:59:31 -0700 Subject: [PATCH 08/10] add test suite for aws signing module --- test/aws_util_SUITE.erl | 56 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 test/aws_util_SUITE.erl diff --git a/test/aws_util_SUITE.erl b/test/aws_util_SUITE.erl new file mode 100644 index 000000000..6b7fbb69c --- /dev/null +++ b/test/aws_util_SUITE.erl @@ -0,0 +1,56 @@ +%%%---------------------------------------------------------------------- +%%% File : aws_util_SUITE.erl +%%% Usage : AWS URL Signing Tests +%%% Author : Roman Hargrave +%%% Created : 08 Sept 2022 by Roman Hargrave +%%% +%%% +%%% This program is free software; you can redistribute it and/or +%%% modify it under the terms of the GNU General Public License as +%%% published by the Free Software Foundation; either version 2 of the +%%% License, or (at your option) any later version. +%%% +%%% This program is distributed in the hope that it will be useful, +%%% but WITHOUT ANY WARRANTY; without even the implied warranty of +%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +%%% General Public License for more details. +%%% +%%% You should have received a copy of the GNU General Public License along +%%% with this program; if not, write to the Free Software Foundation, Inc., +%%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +%%%---------------------------------------------------------------------- + +-module(aws_util_SUITE). +-author("roman@hargrave.info"). + +-include("aws.hrl"). + +-behaviour(ct_suite). + +-export([all/0, + signs_url/1]). + +all() -> + [signs_url]. + +-define(EXPECT_SIGNED_URL, + <<"https://examplebucket.s3.amazonaws.com/test.txt?X-Amz-Signature=aeeed9bbccd4d02ee5c0109b86d86835f995330da4c265957d157751f604d404&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20130524T000000Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host">>). + +% Amazon effectively declares that any valid signing implementation +% should generate the signature +% aeeed9bbccd4d02ee5c0109b86d86835f995330da4c265957d157751f604d404 for +% the values given below +signs_url(_Config) -> + Auth = #aws_auth{access_key_id = <<"AKIAIOSFODNN7EXAMPLE">>, + access_key = <<"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY">>, + region = <<"us-east-1">>}, + Time = {{2013, 05, 24}, {00, 00, 00}}, + TTL = 86400, + URL = <<"https://examplebucket.s3.amazonaws.com/test.txt">>, + case aws_util:signed_url(Auth, get, ?AWS_SERVICE_S3, URL, [], Time, TTL) of + ?EXPECT_SIGNED_URL -> + ok; + UnexpectedUrl -> + {failed, {{expected, ?EXPECT_SIGNED_URL}, + {received, UnexpectedUrl}}} + end. From 4007c806c3c33490c8a3c0601c92443b9f96ad22 Mon Sep 17 00:00:00 2001 From: Roman Hargrave Date: Fri, 9 Sep 2022 23:01:48 -0700 Subject: [PATCH 09/10] style: rename Url to URL --- src/aws_util.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aws_util.erl b/src/aws_util.erl index 31c0388ba..7f4e2f7bf 100644 --- a/src/aws_util.erl +++ b/src/aws_util.erl @@ -57,8 +57,8 @@ ) -> SignedUrl :: binary(). % sign a URL given headers, a verb, authentication details, and a time -signed_url(Auth, Verb, Service, Url, ExtraHeaders, Time, TTL) -> - #{host := Host} = UnauthenticatedUriMap = uri_string:parse(Url), +signed_url(Auth, Verb, Service, URL, ExtraHeaders, Time, TTL) -> + #{host := Host} = UnauthenticatedUriMap = uri_string:parse(URL), Headers = [{<<"host">>, Host} | ExtraHeaders], % insert authentication params. QueryList = sorted_query_list(uri_query_list(UnauthenticatedUriMap) From 1907c26735b2e14493b5d2a6168cce348e866ed4 Mon Sep 17 00:00:00 2001 From: Roman Hargrave Date: Sat, 10 Sep 2022 11:24:00 -0700 Subject: [PATCH 10/10] fix: 21.3 compatibility --- src/aws_util.erl | 13 +++++++------ src/mod_s3_upload.erl | 10 +++++++--- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/aws_util.erl b/src/aws_util.erl index 7f4e2f7bf..c24f126a9 100644 --- a/src/aws_util.erl +++ b/src/aws_util.erl @@ -39,6 +39,7 @@ -import(crypto, [mac/4]). -import(uri_string, [compose_query/1, dissect_query/1]). +-import(misc, [crypto_hmac/3]). -export([signed_url/7]). @@ -67,7 +68,7 @@ signed_url(Auth, Verb, Service, URL, ExtraHeaders, Time, TTL) -> % generate and sign the message StringToSign = string_to_sign(Auth, Time, Service, Verb, UriMap, Headers), SigningKey = signing_key(Auth, Time, Service), - Signature = encode_hex(mac(hmac, sha256, SigningKey, StringToSign)), + Signature = encode_hex(crypto_hmac(sha256, SigningKey, StringToSign)), % add signature to the query list and compose URI SignedQueryString = compose_query([{<<"X-Amz-Signature">>, Signature}|QueryList]), uri_string:recompose(UriMap#{query => SignedQueryString}). @@ -114,7 +115,7 @@ verb(delete) -> EncodedData :: binary(). % lowercase binary:encode_hex encode_hex(Data) -> - str:to_lower(binary:encode_hex(Data)). + str:to_lower(str:to_hexlist(Data)). -spec iso8601_timestamp_utc( DateTime :: calendar:datetime() @@ -249,7 +250,7 @@ signing_key(#aws_auth{access_key = AccessKey, region = Region}, Time, Service) -> - DateKey = mac(hmac, sha256, <<"AWS4", AccessKey/binary>>, iso8601_date(Time)), - DateRegionKey = mac(hmac, sha256, DateKey, Region), - DateRegionServiceKey = mac(hmac, sha256, DateRegionKey, Service), - mac(hmac, sha256, DateRegionServiceKey, <<"aws4_request">>). + DateKey = crypto_hmac(sha256, <<"AWS4", AccessKey/binary>>, iso8601_date(Time)), + DateRegionKey = crypto_hmac(sha256, DateKey, Region), + DateRegionServiceKey = crypto_hmac(sha256, DateRegionKey, Service), + crypto_hmac(sha256, DateRegionServiceKey, <<"aws4_request">>). diff --git a/src/mod_s3_upload.erl b/src/mod_s3_upload.erl index 545ccfb90..7af9a8062 100644 --- a/src/mod_s3_upload.erl +++ b/src/mod_s3_upload.erl @@ -447,12 +447,16 @@ object_url(BucketURL, FileName) -> FileName :: binary() ) -> ObjectName :: binary(). -% generate a unique-in-time object name +% generate a unique-in-time object name. the name consists of a hash +% derived from the file, node, time, and a random number, a +% forward-slash, and the original filename. this ensures that it does +% not collide with other objects, while the forward slash ensures that +% the client displays only the original file name. object_name(FileName) -> MD = crypto:hash_init(sha256), MDFilename = crypto:hash_update(MD, FileName), - MDNodeName = crypto:hash_update(MDFilename, erlang:atom_to_binary(node())), + MDNodeName = crypto:hash_update(MDFilename, atom_to_list(node())), MDTime = crypto:hash_update(MDNodeName, <<(os:system_time())>>), MDRand = crypto:hash_update(MDTime, crypto:strong_rand_bytes(256)), Hash = crypto:hash_final(MDRand), - <<(binary:encode_hex(Hash))/binary, "/", FileName/binary>>. + <<(str:to_hexlist(Hash))/binary, "/", FileName/binary>>.