Skip to content

Commit

Permalink
Add hex escaping of binaries to MySQL backend. (#1678)
Browse files Browse the repository at this point in the history
  • Loading branch information
kzemek authored and fenek committed Jan 26, 2018
1 parent b8326b8 commit b0f8214
Show file tree
Hide file tree
Showing 7 changed files with 85 additions and 84 deletions.
10 changes: 9 additions & 1 deletion src/bin_to_hex.erl
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

-compile([native, {hipe, [o3]}, {inline, [hex/1]}]).

-export([bin_to_hex/1]).
-export([bin_to_hex/1, hex_to_bin/1]).

bin_to_hex(B) when is_binary(B) ->
bin_to_hex(B, <<>>).
Expand All @@ -24,6 +24,14 @@ bin_to_hex_(<<A:8, B:8, C:8, D:8, E:8, F:8, G:8, H:8, Rest/binary>>, Acc) ->
<<Acc/binary,
?H(A), ?H(B), ?H(C), ?H(D), ?H(E), ?H(F), ?H(G), ?H(H)>>).

-spec hex_to_bin(binary()) -> <<_:_*1>>.
hex_to_bin(Bin) when is_binary(Bin) ->
<< <<(hex_to_int(X, Y))>> || <<X, Y>> <= Bin>>.

-spec hex_to_int(byte(), byte()) -> integer().
hex_to_int(X, Y) when is_integer(X), is_integer(Y) ->
list_to_integer([X, Y], 16).

hex(X) ->
element(
X+1, {16#3030, 16#3031, 16#3032, 16#3033, 16#3034, 16#3035, 16#3036,
Expand Down
20 changes: 8 additions & 12 deletions src/mod_mam_muc_odbc_arch.erl
Original file line number Diff line number Diff line change
Expand Up @@ -147,8 +147,7 @@ archive_message_unsafe(Host, MessID, RoomID, FromNick, Packet) ->
SRoomID = integer_to_list(RoomID),
SFromNick = mongoose_rdbms:escape(FromNick),
Data = packet_to_stored_binary(Host, Packet),
EscFormat = mongoose_rdbms:escape_format(Host),
SData = mongoose_rdbms:escape_binary(EscFormat, Data),
SData = mongoose_rdbms:escape_binary(Host, Data),
SMessID = integer_to_list(MessID),
TextBody = mod_mam_utils:packet_to_search_body(mod_mam_muc, Host, Packet),
STextBody = mongoose_rdbms:escape(TextBody),
Expand All @@ -165,7 +164,7 @@ write_message(Host, SMessID, SRoomID, SFromNick, SData, STextBody) ->
["INSERT INTO ", "mam_muc_message", " ",
"(id, room_id, nick_name, message, search_body) "
"VALUES ('", SMessID, "', '", SRoomID, "', "
"'", SFromNick, "', '", SData, "', '", STextBody, "')"]),
"'", SFromNick, "', ", SData, ", '", STextBody, "')"]),
ok.


Expand Down Expand Up @@ -390,18 +389,16 @@ before_id(ID, Filter) ->
-spec rows_to_uniform_format([raw_row()], jid:server(), jid:jid()) ->
[mod_mam_muc:row()].
rows_to_uniform_format(MessageRows, Host, RoomJID) ->
EscFormat = mongoose_rdbms:escape_format(Host),
DbEngine = mongoose_rdbms:db_engine(Host),
[row_to_uniform_format(Host, DbEngine, EscFormat, Row, RoomJID) || Row <- MessageRows].
Pool = mongoose_rdbms_sup:pool(Host),
[row_to_uniform_format(Host, Pool, Row, RoomJID) || Row <- MessageRows].


-spec row_to_uniform_format(jid:server(), atom(), atom(),
raw_row(), jid:jid()) -> mod_mam_muc:row().
row_to_uniform_format(Host, DbEngine, EscFormat, {BMessID, BNick, SDataRaw}, RoomJID) ->
-spec row_to_uniform_format(jid:server(), mongoose_rdbms:pool(), raw_row(), jid:jid()) ->
mod_mam_muc:row().
row_to_uniform_format(Host, Pool, {BMessID, BNick, SDataRaw}, RoomJID) ->
MessID = mongoose_rdbms:result_to_integer(BMessID),
SrcJID = jid:replace_resource(RoomJID, BNick),
SData = mongoose_rdbms:unescape_odbc_binary(DbEngine, SDataRaw),
Data = mongoose_rdbms:unescape_binary(EscFormat, SData),
Data = mongoose_rdbms:unescape_binary(Pool, SDataRaw),
Packet = stored_binary_to_packet(Host, Data),
{MessID, SrcJID, Packet}.

Expand Down Expand Up @@ -646,4 +643,3 @@ stored_binary_to_packet(Host, Bin) ->
-spec db_message_codec(Host :: jid:server()) -> module().
db_message_codec(Host) ->
gen_mod:get_module_opt(Host, ?MODULE, db_message_format, mam_message_compressed_eterm).

18 changes: 7 additions & 11 deletions src/mod_mam_odbc_arch.erl
Original file line number Diff line number Diff line change
Expand Up @@ -227,8 +227,7 @@ do_archive_message(_Result, Host, MessID, UserID,
Data = packet_to_stored_binary(Host, Packet),
TextBody = mod_mam_utils:packet_to_search_body(mod_mam, Host, Packet),
STextBody = mongoose_rdbms:escape(TextBody),
EscFormat = mongoose_rdbms:escape_format(Host),
SData = mongoose_rdbms:escape_binary(EscFormat, Data),
SData = mongoose_rdbms:escape_binary(Host, Data),
SMessID = integer_to_list(MessID),
Table = "mam_message",
write_message(Host, Table, SMessID, SUserID, SBareRemJID,
Expand All @@ -238,7 +237,7 @@ do_archive_message(_Result, Host, MessID, UserID,
-spec write_message(Host :: jid:server(), Table :: string(),
SMessID :: string(), SUserID :: string(), SBareRemJID :: string(),
SRemLResource :: string(), SDir :: string(), SSrcJID :: string(),
SData :: string(), TextBody :: string()) -> 'ok'.
SData :: binary(), TextBody :: string()) -> 'ok'.
write_message(Host, Table, SMessID, SUserID, SBareRemJID,
SRemLResource, SDir, SSrcJID, SData, STextBody) ->
{updated, 1} =
Expand All @@ -249,7 +248,7 @@ write_message(Host, Table, SMessID, SUserID, SBareRemJID,
"from_jid, message, search_body) "
"VALUES ('", SMessID, "', '", SUserID, "', '", SBareRemJID, "', "
"'", SRemLResource, "', '", SDir, "', ",
"'", SSrcJID, "', '", SData, "', '", STextBody, "');"]),
"'", SSrcJID, "', ", SData, ", '", STextBody, "');"]),
ok.

prepare_message(Host, MessID, UserID, LocJID = #jid{}, RemJID = #jid{lresource = RemLResource},
Expand Down Expand Up @@ -483,15 +482,13 @@ before_id(ID, Filter) ->
[Filter, " AND id < '", SID, "'"].

rows_to_uniform_format(Host, UserJID, MessageRows) ->
EscFormat = mongoose_rdbms:escape_format(Host),
DbEngine = mongoose_rdbms:db_engine(Host),
[row_to_uniform_format(Host, DbEngine, UserJID, EscFormat, Row) || Row <- MessageRows].
Pool = mongoose_rdbms_sup:pool(Host),
[row_to_uniform_format(Host, Pool, UserJID, Row) || Row <- MessageRows].

row_to_uniform_format(Host, DbEngine, UserJID, EscFormat, {BMessID, BSrcJID, SDataRaw}) ->
row_to_uniform_format(Host, Pool, UserJID, {BMessID, BSrcJID, SDataRaw}) ->
MessID = mongoose_rdbms:result_to_integer(BMessID),
SrcJID = stored_binary_to_jid(Host, UserJID, BSrcJID),
SData = mongoose_rdbms:unescape_odbc_binary(DbEngine, SDataRaw),
Data = mongoose_rdbms:unescape_binary(EscFormat, SData),
Data = mongoose_rdbms:unescape_binary(Pool, SDataRaw),
Packet = stored_binary_to_packet(Host, Data),
{MessID, SrcJID, Packet}.

Expand Down Expand Up @@ -758,4 +755,3 @@ db_jid_codec(Host) ->
-spec db_message_codec(jid:server()) -> module().
db_message_codec(Host) ->
gen_mod:get_module_opt(Host, ?MODULE, db_message_format, mam_message_compressed_eterm).

50 changes: 14 additions & 36 deletions src/rdbms/mongoose_rdbms.erl
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@

-behaviour(gen_server).

-callback escape_format(Pool :: pool()) -> atom().
-callback escape_binary(Pool :: pool(), binary()) -> iodata().
-callback unescape_binary(Pool :: pool(), binary()) -> binary().
-callback connect(Args :: any(), QueryTimeout :: non_neg_integer()) ->
{ok, Connection :: term()} | {error, Reason :: any()}.
-callback disconnect(Connection :: term()) -> any().
Expand All @@ -56,10 +57,8 @@
print_state/1]).

%% BLOB escaping
-export([escape_format/1,
escape_binary/2,
unescape_binary/2,
unescape_odbc_binary/2]).
-export([escape_binary/2,
unescape_binary/2]).

%% count / integra types decoding
-export([result_to_integer/1]).
Expand Down Expand Up @@ -213,45 +212,24 @@ escape_like(S) ->
rdbms_queries:escape_like_string(S).


-spec escape_format(server()) -> hex | simple_escape.
escape_format(HostOrPool) ->
-spec escape_binary(server(), binary()) -> iodata().
escape_binary(HostOrPool, Bin) when is_binary(Bin) ->
Pool = mongoose_rdbms_sup:pool(HostOrPool),
mongoose_rdbms_backend:escape_format(Pool).

-spec escape_binary('hex' | 'simple_escape', binary()) -> binary() | string().
escape_binary(hex, Bin) when is_binary(Bin) ->
<<"\\\\x", (bin_to_hex:bin_to_hex(Bin))/binary>>;
escape_binary(mssql_hex, Bin) when is_binary(Bin) ->
bin_to_hex:bin_to_hex(Bin);
escape_binary(simple_escape, Bin) when is_binary(Bin) ->
escape(Bin).

-spec unescape_binary('hex' | 'simple_escape', binary()) -> binary().
unescape_binary(hex, <<"\\x", Bin/binary>>) when is_binary(Bin) ->
hex_to_bin(Bin);
unescape_binary(_, Bin) ->
Bin.

-spec unescape_odbc_binary(atom(), binary()) -> binary().
unescape_odbc_binary(odbc, Bin) when is_binary(Bin)->
hex_to_bin(Bin);
unescape_odbc_binary(_, Bin) ->
Bin.
mongoose_rdbms_backend:escape_binary(Pool, Bin).


-spec unescape_binary(server(), binary()) -> binary().
unescape_binary(HostOrPool, Bin) when is_binary(Bin) ->
Pool = mongoose_rdbms_sup:pool(HostOrPool),
mongoose_rdbms_backend:unescape_binary(Pool, Bin).


-spec result_to_integer(binary() | integer()) -> integer().
result_to_integer(Int) when is_integer(Int) ->
Int;
result_to_integer(Bin) when is_binary(Bin) ->
binary_to_integer(Bin).

-spec hex_to_bin(binary()) -> <<_:_*1>>.
hex_to_bin(Bin) when is_binary(Bin) ->
<< <<(hex_to_int(X, Y))>> || <<X, Y>> <= Bin>>.

-spec hex_to_int(byte(), byte()) -> integer().
hex_to_int(X, Y) when is_integer(X), is_integer(Y) ->
list_to_integer([X, Y], 16).

-spec to_bool(binary() | string() | atom() | integer() | any()) -> boolean().
to_bool(B) when is_binary(B) ->
to_bool(binary_to_list(B));
Expand Down
13 changes: 9 additions & 4 deletions src/rdbms/mongoose_rdbms_mysql.erl
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,18 @@

-define(MYSQL_PORT, 3306).

-export([escape_format/1, connect/2, disconnect/1, query/3, prepare/6, execute/4]).
-export([escape_binary/2, unescape_binary/2, connect/2, disconnect/1,
query/3, prepare/6, execute/4]).

%% API

-spec escape_format(mongoose_rdbms:pool()) -> atom().
escape_format(_Pool) ->
simple_escape.
-spec escape_binary(mongoose_rdbms:pool(), binary()) -> iodata().
escape_binary(_Pool, Bin) when is_binary(Bin) ->
[<<"X'">>, bin_to_hex:bin_to_hex(Bin), <<"'">>].

-spec unescape_binary(mongoose_rdbms:pool(), binary()) -> binary().
unescape_binary(_Pool, Bin) when is_binary(Bin) ->
Bin.

-spec connect(Args :: any(), QueryTimeout :: non_neg_integer()) ->
{ok, Connection :: term()} | {error, Reason :: any()}.
Expand Down
43 changes: 27 additions & 16 deletions src/rdbms/mongoose_rdbms_odbc.erl
Original file line number Diff line number Diff line change
Expand Up @@ -18,20 +18,18 @@
-author('[email protected]').
-behaviour(mongoose_rdbms).

-export([escape_format/1, connect/2, disconnect/1, query/3, prepare/6, execute/4]).
-export([escape_binary/2, unescape_binary/2, connect/2, disconnect/1,
query/3, prepare/6, execute/4]).

%% API

-spec escape_format(mongoose_rdbms:pool()) -> atom().
escape_format(Pool) ->
case mongoose_rdbms_sup:get_option(Pool, odbc_server_type) of
pgsql ->
hex;
mssql ->
mssql_hex;
_ ->
simple_escape
end.
-spec escape_binary(mongoose_rdbms:pool(), binary()) -> iodata().
escape_binary(Pool, Bin) when is_binary(Bin) ->
escape_binary(Pool, server_type(Pool), Bin).

-spec unescape_binary(mongoose_rdbms:pool(), binary()) -> binary().
unescape_binary(_Pool, Bin) when is_binary(Bin) ->
bin_to_hex:hex_to_bin(Bin).

-spec connect(Args :: any(), QueryTimeout :: non_neg_integer()) ->
{ok, Connection :: term()} | {error, Reason :: any()}.
Expand Down Expand Up @@ -61,10 +59,10 @@ query(Connection, Query, Timeout) ->
Fields :: [binary()], Statement :: iodata()) ->
{ok, {[binary()], [fun((term()) -> tuple())]}}.
prepare(Pool, Connection, _Name, Table, Fields, Statement) ->
BinEscapeFormat = escape_format(Pool),
{ok, TableDesc} = odbc:describe_table(Connection, unicode:characters_to_list(Table)),
SplitQuery = binary:split(iolist_to_binary(Statement), <<"?">>, [global]),
ParamMappers = [field_name_to_mapper(BinEscapeFormat, TableDesc, Field) || Field <- Fields],
ServerType = server_type(Pool),
ParamMappers = [field_name_to_mapper(Pool, ServerType, TableDesc, Field) || Field <- Fields],
{ok, {SplitQuery, ParamMappers}}.

-spec execute(Connection :: term(), Statement :: {[binary()], [fun((term()) -> tuple())]},
Expand All @@ -89,13 +87,14 @@ parse({error, Reason}) ->
parse(Other) ->
Other.

-spec field_name_to_mapper(BinEscFormat :: atom(), TableDesc :: proplists:proplist(),
-spec field_name_to_mapper(Pool :: mongoose_rdbms:pool(), ServerType :: atom(),
TableDesc :: proplists:proplist(),
FieldName :: binary()) -> fun((term()) -> tuple()).
field_name_to_mapper(BinEscFormat, TableDesc, FieldName) ->
field_name_to_mapper(Pool, ServerType, TableDesc, FieldName) ->
{_, ODBCType} = lists:keyfind(unicode:characters_to_list(FieldName), 1, TableDesc),
case ODBCType of
T when T =:= 'SQL_BINARY'; T =:= 'SQL_VARBINARY'; T =:= 'SQL_LONGVARBINARY' ->
fun(P) -> {[<<"'">>, mongoose_rdbms:escape_binary(BinEscFormat, P), <<"'">>], []} end;
fun(P) -> {[escape_binary(Pool, ServerType, P)], []} end;
'SQL_LONGVARCHAR' ->
fun(P) -> {[<<"'">>, mongoose_rdbms:escape(P), <<"'">>], []} end;
'SQL_BIGINT' ->
Expand Down Expand Up @@ -123,3 +122,15 @@ unsplit_query([QueryHead | QueryRest], ParamMappers, [Param | Params], QueryAcc,
NewQueryAcc = [InlineQuery, QueryHead | QueryAcc],
NewParamsAcc = ODBCParam ++ ParamsAcc,
unsplit_query(QueryRest, NextParamMappers, Params, NewQueryAcc, NewParamsAcc).

-spec server_type(mongoose_rdbms:pool()) -> atom().
server_type(Pool) ->
mongoose_rdbms_sup:get_option(Pool, odbc_server_type).

-spec escape_binary(mongoose_rdbms:pool(), ServerType :: atom(), binary()) -> iodata().
escape_binary(Pool, pgsql, Bin) ->
mongoose_rdbms_pgsql:escape_binary(Pool, Bin);
escape_binary(Pool, mysql, Bin) ->
mongoose_rdbms_mysql:escape_binary(Pool, Bin);
escape_binary(_Pool, _ServerType, Bin) ->
[$', bin_to_hex:bin_to_hex(Bin), $'].
15 changes: 11 additions & 4 deletions src/rdbms/mongoose_rdbms_pgsql.erl
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,20 @@

-define(PGSQL_PORT, 5432).

-export([escape_format/1, connect/2, disconnect/1, query/3, prepare/6, execute/4]).
-export([escape_binary/2, unescape_binary/2, connect/2, disconnect/1,
query/3, prepare/6, execute/4]).

%% API

-spec escape_format(mongoose_rdbms:pool()) -> atom().
escape_format(_Pool) ->
hex.
-spec escape_binary(mongoose_rdbms:pool(), binary()) -> iodata().
escape_binary(_Pool, Bin) when is_binary(Bin) ->
[<<"E'\\\\x">>, bin_to_hex:bin_to_hex(Bin), <<"'::bytea">>].

-spec unescape_binary(mongoose_rdbms:pool(), binary()) -> binary().
unescape_binary(_Pool, <<"\\x", Bin/binary>>) ->
bin_to_hex:hex_to_bin(Bin);
unescape_binary(_Pool, Bin) when is_binary(Bin) ->
Bin.

-spec connect(Args :: any(), QueryTimeout :: non_neg_integer()) ->
{ok, Connection :: term()} | {error, Reason :: any()}.
Expand Down

0 comments on commit b0f8214

Please sign in to comment.