From 0b31aa490ba19b85c164fcd463131979ff2ac1c7 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Pawe=C5=82=20Chmielowski?= Date: Wed, 28 Nov 2018 11:25:04 +0100 Subject: [PATCH] Add xml compression to sql backend of mam --- src/mod_mam.erl | 3 +- src/mod_mam_sql.erl | 21 +- src/xml_compress.erl | 958 +++++++++++++++++++++++++++++++++++++ tools/xml_compress_gen.erl | 417 ++++++++++++++++ 4 files changed, 1392 insertions(+), 7 deletions(-) create mode 100644 src/xml_compress.erl create mode 100644 tools/xml_compress_gen.erl diff --git a/src/mod_mam.erl b/src/mod_mam.erl index 9689a93a2..58108fb1e 100644 --- a/src/mod_mam.erl +++ b/src/mod_mam.erl @@ -1172,7 +1172,7 @@ mod_opt_type(O) when O == cache_life_time; O == cache_size -> fun (I) when is_integer(I), I > 0 -> I; (infinity) -> infinity end; -mod_opt_type(O) when O == use_cache; O == cache_missed -> +mod_opt_type(O) when O == use_cache; O == cache_missed; O == compress_xml -> fun (B) when is_boolean(B) -> B end; mod_opt_type(db_type) -> fun(T) -> ejabberd_config:v_db(?MODULE, T) end; mod_opt_type(default) -> @@ -1187,6 +1187,7 @@ mod_options(Host) -> [{assume_mam_usage, false}, {default, never}, {request_activates_archiving, false}, + {compress_xml, false}, {db_type, ejabberd_config:default_db(Host, ?MODULE)}, {use_cache, ejabberd_config:use_cache(Host)}, {cache_size, ejabberd_config:cache_size(Host)}, diff --git a/src/mod_mam_sql.erl b/src/mod_mam_sql.erl index 37ea8dc6f..1c9b7cea2 100644 --- a/src/mod_mam_sql.erl +++ b/src/mod_mam_sql.erl @@ -102,9 +102,18 @@ store(Pkt, LServer, {LUser, LHost}, Type, Peer, Nick, _Dir, TS) -> jid:remove_resource(Peer))), LPeer = jid:encode( jid:tolower(Peer)), - XML = fxml:element_to_binary(Pkt), Body = fxml:get_subtag_cdata(Pkt, <<"body">>), SType = misc:atom_to_binary(Type), + XML = case gen_mod:get_module_opt(LServer, mod_mam, compress_xml) of + true -> + J1 = case Type of + chat -> jid:encode({LUser, LHost, <<>>}); + groupchat -> SUser + end, + xml_compress:encode(Pkt, J1, LPeer); + _ -> + fxml:element_to_binary(Pkt) + end, case ejabberd_sql:sql_query( LServer, ?SQL_INSERT( @@ -192,8 +201,8 @@ select(LServer, JidRequestor, #jid{luser = LUser} = JidArchive, {lists:flatmap( fun([TS, XML, PeerBin, Kind, Nick]) -> case make_archive_el( - TS, XML, PeerBin, Kind, Nick, - MsgType, JidRequestor, JidArchive) of + jid:encode(JidArchive), TS, XML, PeerBin, Kind, Nick, + MsgType, JidRequestor, JidArchive) of {ok, El} -> [{TS, binary_to_integer(TS), El}]; {error, _} -> @@ -399,13 +408,13 @@ get_max_direction_id(RSM) -> {undefined, undefined, <<>>} end. --spec make_archive_el(binary(), binary(), binary(), binary(), +-spec make_archive_el(binary(), binary(), binary(), binary(), binary(), binary(), _, jid(), jid()) -> {ok, xmpp_element()} | {error, invalid_jid | invalid_timestamp | invalid_xml}. -make_archive_el(TS, XML, Peer, Kind, Nick, MsgType, JidRequestor, JidArchive) -> - case fxml_stream:parse_element(XML) of +make_archive_el(User, TS, XML, Peer, Kind, Nick, MsgType, JidRequestor, JidArchive) -> + case xml_compress:decode(XML, User, Peer) of #xmlel{} = El -> try binary_to_integer(TS) of TSInt -> diff --git a/src/xml_compress.erl b/src/xml_compress.erl new file mode 100644 index 000000000..673b25c14 --- /dev/null +++ b/src/xml_compress.erl @@ -0,0 +1,958 @@ +-module(xml_compress). +-export([encode/3, decode/3]). + +% This file was generated by xml_compress_gen +% +% Rules used: +% +% [{<<"eu.siacs.conversations.axolotl">>,<<"key">>, +% [{<<"prekey">>,[<<"true">>]},{<<"rid">>,[]}], +% []}, +% {<<"jabber:client">>,<<"message">>, +% [{<<"from">>,[j2,{j1}]}, +% {<<"id">>,[]}, +% {<<"to">>,[j1,j2,{j1}]}, +% {<<"type">>,[<<"chat">>,<<"groupchat">>,<<"normal">>]}, +% {<<"xml:lang">>,[<<"en">>]}], +% []}, +% {<<"urn:xmpp:hints">>,<<"store">>,[],[]}, +% {<<"jabber:client">>,<<"body">>,[], +% [<<73,32,115,101,110,116,32,121,111,117,32,97,110,32,79,77,69,77,79,32,101, +% 110,99,114,121,112,116,101,100,32,109,101,115,115,97,103,101,32,98,117, +% 116,32,121,111,117,114,32,99,108,105,101,110,116,32,100,111,101,115,110, +% 226,128,153,116,32,115,101,101,109,32,116,111,32,115,117,112,112,111, +% 114,116,32,116,104,97,116,46,32,70,105,110,100,32,109,111,114,101,32, +% 105,110,102,111,114,109,97,116,105,111,110,32,111,110,32,104,116,116, +% 112,115,58,47,47,99,111,110,118,101,114,115,97,116,105,111,110,115,46, +% 105,109,47,111,109,101,109,111>>]}, +% {<<"urn:xmpp:sid:0">>,<<"origin-id">>,[{<<"id">>,[]}],[]}, +% {<<"urn:xmpp:chat-markers:0">>,<<"markable">>,[],[]}, +% {<<"eu.siacs.conversations.axolotl">>,<<"encrypted">>,[],[]}, +% {<<"eu.siacs.conversations.axolotl">>,<<"header">>,[{<<"sid">>,[]}],[]}, +% {<<"eu.siacs.conversations.axolotl">>,<<"iv">>,[],[]}, +% {<<"eu.siacs.conversations.axolotl">>,<<"payload">>,[],[]}, +% {<<"urn:xmpp:eme:0">>,<<"encryption">>, +% [{<<"name">>,[<<"OMEMO">>]}, +% {<<"namespace">>,[<<"eu.siacs.conversations.axolotl">>]}], +% []}, +% {<<"urn:xmpp:delay">>,<<"delay">>,[{<<"from">>,[j1]},{<<"stamp">>,[]}],[]}, +% {<<"http://jabber.org/protocol/address">>,<<"address">>, +% [{<<"jid">>,[{j1}]},{<<"type">>,[<<"ofrom">>]}], +% []}, +% {<<"http://jabber.org/protocol/address">>,<<"addresses">>,[],[]}, +% {<<"urn:xmpp:chat-markers:0">>,<<"displayed">>, +% [{<<"id">>,[]},{<<"sender">>,[{j1},{j2}]}], +% []}, +% {<<"urn:xmpp:mam:tmp">>,<<"archived">>,[{<<"by">>,[]},{<<"id">>,[]}],[]}, +% {<<"urn:xmpp:sid:0">>,<<"stanza-id">>,[{<<"by">>,[]},{<<"id">>,[]}],[]}, +% {<<"urn:xmpp:receipts">>,<<"request">>,[],[]}, +% {<<"urn:xmpp:chat-markers:0">>,<<"received">>,[{<<"id">>,[]}],[]}, +% {<<"urn:xmpp:receipts">>,<<"received">>,[{<<"id">>,[]}],[]}, +% {<<"http://jabber.org/protocol/chatstates">>,<<"active">>,[],[]}, +% {<<"http://jabber.org/protocol/muc#user">>,<<"invite">>, +% [{<<"from">>,[{j1}]}], +% []}, +% {<<"http://jabber.org/protocol/muc#user">>,<<"reason">>,[],[]}, +% {<<"http://jabber.org/protocol/muc#user">>,<<"x">>,[],[]}, +% {<<"jabber:x:conference">>,<<"x">>,[{<<"jid">>,[j2]}],[]}, +% {<<"jabber:client">>,<<"subject">>,[],[]}, +% {<<"jabber:client">>,<<"thread">>,[],[]}, +% {<<"http://jabber.org/protocol/pubsub#event">>,<<"event">>,[],[]}, +% {<<"http://jabber.org/protocol/pubsub#event">>,<<"item">>,[{<<"id">>,[]}],[]}, +% {<<"http://jabber.org/protocol/pubsub#event">>,<<"items">>, +% [{<<"node">>,[<<"urn:xmpp:mucsub:nodes:messages">>]}], +% []}, +% {<<"p1:push:custom">>,<<"x">>,[{<<"key">>,[]},{<<"value">>,[]}],[]}, +% {<<"p1:pushed">>,<<"x">>,[],[]}, +% {<<"urn:xmpp:message-correct:0">>,<<"replace">>,[{<<"id">>,[]}],[]}, +% {<<"http://jabber.org/protocol/chatstates">>,<<"composing">>,[],[]}] + +encode(El, J1, J2) -> + encode_child(El, <<"jabber:client">>, + J1, J2, byte_size(J1), byte_size(J2), <<1:8>>). + +encode_attr({<<"xmlns">>, _}, Acc) -> + Acc; +encode_attr({N, V}, Acc) -> + <>. + +encode_attrs(Attrs, Acc) -> + lists:foldl(fun encode_attr/2, Acc, Attrs). + +encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) -> + E1 = if + PNs == Ns -> encode_attrs(Attrs, <>); + true -> encode_attrs(Attrs, <>) + end, + E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>), + <>. + +encode_child({xmlel, Name, Attrs, Children}, PNs, J1, J2, J1L, J2L, Pfx) -> + case lists:keyfind(<<"xmlns">>, 1, Attrs) of + false -> + encode(PNs, PNs, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx); + {_, Ns} -> + encode(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) + end; +encode_child({xmlcdata, Data}, _PNs, _J1, _J2, _J1L, _J2L, Pfx) -> + <>. + +encode_children(Children, PNs, J1, J2, J1L, J2L, Pfx) -> + lists:foldl( + fun(Child, Acc) -> + encode_child(Child, PNs, J1, J2, J1L, J2L, Acc) + end, Pfx, Children). + +encode_string(Data) -> + <> = <<(byte_size(Data)):16/unsigned-big-integer>>, + case {V1, V2, V3} of + {0, 0, V3} -> + <>; + {0, V2, V3} -> + <<(V3 bor 64):8, V2:8, Data/binary>>; + _ -> + <<(V3 bor 64):8, (V2 bor 64):8, V1:8, Data/binary>> + end. + +encode(PNs, <<"eu.siacs.conversations.axolotl">> = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) -> + case Name of + <<"key">> -> + E = lists:foldl(fun + ({<<"prekey">>, AVal}, Acc) -> + case AVal of + <<"true">> -> <>; + _ -> <> + end; + ({<<"rid">>, AVal}, Acc) -> + <>; + (Attr, Acc) -> encode_attr(Attr, Acc) + end, <>, Attrs), + E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>), + <>; + <<"encrypted">> -> + E = encode_attrs(Attrs, <>), + E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>), + <>; + <<"header">> -> + E = lists:foldl(fun + ({<<"sid">>, AVal}, Acc) -> + <>; + (Attr, Acc) -> encode_attr(Attr, Acc) + end, <>, Attrs), + E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>), + <>; + <<"iv">> -> + E = encode_attrs(Attrs, <>), + E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>), + <>; + <<"payload">> -> + E = encode_attrs(Attrs, <>), + E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>), + <>; + _ -> encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) +end; +encode(PNs, <<"jabber:client">> = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) -> + case Name of + <<"message">> -> + E = lists:foldl(fun + ({<<"from">>, AVal}, Acc) -> + case AVal of + J2 -> <>; + <> -> <>; + _ -> <> + end; + ({<<"id">>, AVal}, Acc) -> + <>; + ({<<"to">>, AVal}, Acc) -> + case AVal of + J1 -> <>; + J2 -> <>; + <> -> <>; + _ -> <> + end; + ({<<"type">>, AVal}, Acc) -> + case AVal of + <<"chat">> -> <>; + <<"groupchat">> -> <>; + <<"normal">> -> <>; + _ -> <> + end; + ({<<"xml:lang">>, AVal}, Acc) -> + case AVal of + <<"en">> -> <>; + _ -> <> + end; + (Attr, Acc) -> encode_attr(Attr, Acc) + end, <>, Attrs), + E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>), + <>; + <<"body">> -> + E = encode_attrs(Attrs, <>), + E2 = lists:foldl(fun + ({xmlcdata, <<73,32,115,101,110,116,32,121,111,117,32,97,110,32,79,77,69, + 77,79,32,101,110,99,114,121,112,116,101,100,32,109,101,115, + 115,97,103,101,32,98,117,116,32,121,111,117,114,32,99,108, + 105,101,110,116,32,100,111,101,115,110,226,128,153,116,32, + 115,101,101,109,32,116,111,32,115,117,112,112,111,114,116,32, + 116,104,97,116,46,32,70,105,110,100,32,109,111,114,101,32, + 105,110,102,111,114,109,97,116,105,111,110,32,111,110,32,104, + 116,116,112,115,58,47,47,99,111,110,118,101,114,115,97,116, + 105,111,110,115,46,105,109,47,111,109,101,109,111>>}, Acc) -> <>; + (El, Acc) -> encode_child(El, Ns, J1, J2, J1L, J2L, Acc) + end, <>, Children), + <>; + <<"subject">> -> + E = encode_attrs(Attrs, <>), + E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>), + <>; + <<"thread">> -> + E = encode_attrs(Attrs, <>), + E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>), + <>; + _ -> encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) +end; +encode(PNs, <<"urn:xmpp:hints">> = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) -> + case Name of + <<"store">> -> + E = encode_attrs(Attrs, <>), + E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>), + <>; + _ -> encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) +end; +encode(PNs, <<"urn:xmpp:sid:0">> = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) -> + case Name of + <<"origin-id">> -> + E = lists:foldl(fun + ({<<"id">>, AVal}, Acc) -> + <>; + (Attr, Acc) -> encode_attr(Attr, Acc) + end, <>, Attrs), + E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>), + <>; + <<"stanza-id">> -> + E = lists:foldl(fun + ({<<"by">>, AVal}, Acc) -> + <>; + ({<<"id">>, AVal}, Acc) -> + <>; + (Attr, Acc) -> encode_attr(Attr, Acc) + end, <>, Attrs), + E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>), + <>; + _ -> encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) +end; +encode(PNs, <<"urn:xmpp:chat-markers:0">> = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) -> + case Name of + <<"markable">> -> + E = encode_attrs(Attrs, <>), + E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>), + <>; + <<"displayed">> -> + E = lists:foldl(fun + ({<<"id">>, AVal}, Acc) -> + <>; + ({<<"sender">>, AVal}, Acc) -> + case AVal of + <> -> <>; + <> -> <>; + _ -> <> + end; + (Attr, Acc) -> encode_attr(Attr, Acc) + end, <>, Attrs), + E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>), + <>; + <<"received">> -> + E = lists:foldl(fun + ({<<"id">>, AVal}, Acc) -> + <>; + (Attr, Acc) -> encode_attr(Attr, Acc) + end, <>, Attrs), + E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>), + <>; + _ -> encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) +end; +encode(PNs, <<"urn:xmpp:eme:0">> = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) -> + case Name of + <<"encryption">> -> + E = lists:foldl(fun + ({<<"name">>, AVal}, Acc) -> + case AVal of + <<"OMEMO">> -> <>; + _ -> <> + end; + ({<<"namespace">>, AVal}, Acc) -> + case AVal of + <<"eu.siacs.conversations.axolotl">> -> <>; + _ -> <> + end; + (Attr, Acc) -> encode_attr(Attr, Acc) + end, <>, Attrs), + E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>), + <>; + _ -> encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) +end; +encode(PNs, <<"urn:xmpp:delay">> = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) -> + case Name of + <<"delay">> -> + E = lists:foldl(fun + ({<<"from">>, AVal}, Acc) -> + case AVal of + J1 -> <>; + _ -> <> + end; + ({<<"stamp">>, AVal}, Acc) -> + <>; + (Attr, Acc) -> encode_attr(Attr, Acc) + end, <>, Attrs), + E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>), + <>; + _ -> encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) +end; +encode(PNs, <<"http://jabber.org/protocol/address">> = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) -> + case Name of + <<"address">> -> + E = lists:foldl(fun + ({<<"jid">>, AVal}, Acc) -> + case AVal of + <> -> <>; + _ -> <> + end; + ({<<"type">>, AVal}, Acc) -> + case AVal of + <<"ofrom">> -> <>; + _ -> <> + end; + (Attr, Acc) -> encode_attr(Attr, Acc) + end, <>, Attrs), + E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>), + <>; + <<"addresses">> -> + E = encode_attrs(Attrs, <>), + E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>), + <>; + _ -> encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) +end; +encode(PNs, <<"urn:xmpp:mam:tmp">> = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) -> + case Name of + <<"archived">> -> + E = lists:foldl(fun + ({<<"by">>, AVal}, Acc) -> + <>; + ({<<"id">>, AVal}, Acc) -> + <>; + (Attr, Acc) -> encode_attr(Attr, Acc) + end, <>, Attrs), + E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>), + <>; + _ -> encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) +end; +encode(PNs, <<"urn:xmpp:receipts">> = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) -> + case Name of + <<"request">> -> + E = encode_attrs(Attrs, <>), + E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>), + <>; + <<"received">> -> + E = lists:foldl(fun + ({<<"id">>, AVal}, Acc) -> + <>; + (Attr, Acc) -> encode_attr(Attr, Acc) + end, <>, Attrs), + E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>), + <>; + _ -> encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) +end; +encode(PNs, <<"http://jabber.org/protocol/chatstates">> = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) -> + case Name of + <<"active">> -> + E = encode_attrs(Attrs, <>), + E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>), + <>; + <<"composing">> -> + E = encode_attrs(Attrs, <>), + E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>), + <>; + _ -> encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) +end; +encode(PNs, <<"http://jabber.org/protocol/muc#user">> = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) -> + case Name of + <<"invite">> -> + E = lists:foldl(fun + ({<<"from">>, AVal}, Acc) -> + case AVal of + <> -> <>; + _ -> <> + end; + (Attr, Acc) -> encode_attr(Attr, Acc) + end, <>, Attrs), + E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>), + <>; + <<"reason">> -> + E = encode_attrs(Attrs, <>), + E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>), + <>; + <<"x">> -> + E = encode_attrs(Attrs, <>), + E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>), + <>; + _ -> encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) +end; +encode(PNs, <<"jabber:x:conference">> = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) -> + case Name of + <<"x">> -> + E = lists:foldl(fun + ({<<"jid">>, AVal}, Acc) -> + case AVal of + J2 -> <>; + _ -> <> + end; + (Attr, Acc) -> encode_attr(Attr, Acc) + end, <>, Attrs), + E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>), + <>; + _ -> encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) +end; +encode(PNs, <<"http://jabber.org/protocol/pubsub#event">> = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) -> + case Name of + <<"event">> -> + E = encode_attrs(Attrs, <>), + E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>), + <>; + <<"item">> -> + E = lists:foldl(fun + ({<<"id">>, AVal}, Acc) -> + <>; + (Attr, Acc) -> encode_attr(Attr, Acc) + end, <>, Attrs), + E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>), + <>; + <<"items">> -> + E = lists:foldl(fun + ({<<"node">>, AVal}, Acc) -> + case AVal of + <<"urn:xmpp:mucsub:nodes:messages">> -> <>; + _ -> <> + end; + (Attr, Acc) -> encode_attr(Attr, Acc) + end, <>, Attrs), + E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>), + <>; + _ -> encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) +end; +encode(PNs, <<"p1:push:custom">> = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) -> + case Name of + <<"x">> -> + E = lists:foldl(fun + ({<<"key">>, AVal}, Acc) -> + <>; + ({<<"value">>, AVal}, Acc) -> + <>; + (Attr, Acc) -> encode_attr(Attr, Acc) + end, <>, Attrs), + E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>), + <>; + _ -> encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) +end; +encode(PNs, <<"p1:pushed">> = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) -> + case Name of + <<"x">> -> + E = encode_attrs(Attrs, <>), + E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>), + <>; + _ -> encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) +end; +encode(PNs, <<"urn:xmpp:message-correct:0">> = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) -> + case Name of + <<"replace">> -> + E = lists:foldl(fun + ({<<"id">>, AVal}, Acc) -> + <>; + (Attr, Acc) -> encode_attr(Attr, Acc) + end, <>, Attrs), + E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>), + <>; + _ -> encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) +end; +encode(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) -> + encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx). + +decode(<<$<, _/binary>> = Data, _J1, _J2) -> + fxml_stream:parse_element(Data); +decode(<<1:8, Rest/binary>>, J1, J2) -> + {El, _} = decode(Rest, <<"jabber:client">>, J1, J2), + El. + +decode_string(Data) -> + case Data of + <<0:2, L:6, Str:L/binary, Rest/binary>> -> + {Str, Rest}; + <<1:2, L1:6, 0:2, L2:6, Rest/binary>> -> + L = L2*64 + L1, + <> = Rest, + {Str, Rest2}; + <<1:2, L1:6, 1:2, L2:6, L3:8, Rest/binary>> -> + L = (L3*64 + L2)*64 + L1, + <> = Rest, + {Str, Rest2} + end. + +decode_child(<<1:8, Rest/binary>>, _PNs, _J1, _J2) -> + {Text, Rest2} = decode_string(Rest), + {{xmlcdata, Text}, Rest2}; +decode_child(<<2:8, Rest/binary>>, PNs, J1, J2) -> + {Name, Rest2} = decode_string(Rest), + {Attrs, Rest3} = decode_attrs(Rest2), + {Children, Rest4} = decode_children(Rest3, PNs, J1, J2), + {{xmlel, Name, Attrs, Children}, Rest4}; +decode_child(<<3:8, Rest/binary>>, PNs, J1, J2) -> + {Name, Rest2} = decode_string(Rest), + {Ns, Rest3} = decode_string(Rest2), + {Attrs, Rest4} = decode_attrs(Rest3), + {Children, Rest5} = decode_children(Rest4, Ns, J1, J2), + {{xmlel, Name, add_ns(PNs, Ns, Attrs), Children}, Rest5}; +decode_child(<<4:8, Rest/binary>>, _PNs, _J1, _J2) -> + {stop, Rest}; +decode_child(Other, PNs, J1, J2) -> + decode(Other, PNs, J1, J2). + +decode_children(Data, PNs, J1, J2) -> + prefix_map(fun(Data2) -> decode(Data2, PNs, J1, J2) end, Data). + +decode_attr(<<1:8, Rest/binary>>) -> + {Name, Rest2} = decode_string(Rest), + {Val, Rest3} = decode_string(Rest2), + {{Name, Val}, Rest3}; +decode_attr(<<2:8, Rest/binary>>) -> + {stop, Rest}. + +decode_attrs(Data) -> + prefix_map(fun decode_attr/1, Data). + +prefix_map(F, Data) -> + prefix_map(F, Data, []). + +prefix_map(F, Data, Acc) -> + case F(Data) of + {stop, Rest} -> + {lists:reverse(Acc), Rest}; + {Val, Rest} -> + prefix_map(F, Rest, [Val | Acc]) + end. + +add_ns(Ns, Ns, Attrs) -> + Attrs; +add_ns(_, Ns, Attrs) -> + [{<<"xmlns">>, Ns} | Attrs]. + +decode(<<5:8, Rest/binary>>, PNs, J1, J2) -> + Ns = <<"eu.siacs.conversations.axolotl">>, + {Attrs, Rest2} = prefix_map(fun + (<<3:8, Rest3/binary>>) -> + {{<<"prekey">>, <<"true">>}, Rest3}; + (<<4:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"prekey">>, AVal}, Rest4}; + (<<5:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"rid">>, AVal}, Rest4}; + (<<2:8, Rest3/binary>>) -> + {stop, Rest3}; + (Data) -> + decode_attr(Data) + end, Rest), + {Children, Rest6} = decode_children(Rest2, Ns, J1, J2), + {{xmlel, <<"key">>, add_ns(PNs, Ns, Attrs), Children}, Rest6}; +decode(<<12:8, Rest/binary>>, PNs, J1, J2) -> + Ns = <<"eu.siacs.conversations.axolotl">>, + {Attrs, Rest2} = decode_attrs(Rest), + {Children, Rest6} = decode_children(Rest2, Ns, J1, J2), + {{xmlel, <<"encrypted">>, add_ns(PNs, Ns, Attrs), Children}, Rest6}; +decode(<<13:8, Rest/binary>>, PNs, J1, J2) -> + Ns = <<"eu.siacs.conversations.axolotl">>, + {Attrs, Rest2} = prefix_map(fun + (<<3:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"sid">>, AVal}, Rest4}; + (<<2:8, Rest3/binary>>) -> + {stop, Rest3}; + (Data) -> + decode_attr(Data) + end, Rest), + {Children, Rest6} = decode_children(Rest2, Ns, J1, J2), + {{xmlel, <<"header">>, add_ns(PNs, Ns, Attrs), Children}, Rest6}; +decode(<<14:8, Rest/binary>>, PNs, J1, J2) -> + Ns = <<"eu.siacs.conversations.axolotl">>, + {Attrs, Rest2} = decode_attrs(Rest), + {Children, Rest6} = decode_children(Rest2, Ns, J1, J2), + {{xmlel, <<"iv">>, add_ns(PNs, Ns, Attrs), Children}, Rest6}; +decode(<<15:8, Rest/binary>>, PNs, J1, J2) -> + Ns = <<"eu.siacs.conversations.axolotl">>, + {Attrs, Rest2} = decode_attrs(Rest), + {Children, Rest6} = decode_children(Rest2, Ns, J1, J2), + {{xmlel, <<"payload">>, add_ns(PNs, Ns, Attrs), Children}, Rest6}; +decode(<<6:8, Rest/binary>>, PNs, J1, J2) -> + Ns = <<"jabber:client">>, + {Attrs, Rest2} = prefix_map(fun + (<<3:8, Rest3/binary>>) -> + {{<<"from">>, J2}, Rest3}; + (<<4:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"from">>, <>}, Rest4}; + (<<5:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"from">>, AVal}, Rest4}; + (<<6:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"id">>, AVal}, Rest4}; + (<<7:8, Rest3/binary>>) -> + {{<<"to">>, J1}, Rest3}; + (<<8:8, Rest3/binary>>) -> + {{<<"to">>, J2}, Rest3}; + (<<9:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"to">>, <>}, Rest4}; + (<<10:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"to">>, AVal}, Rest4}; + (<<11:8, Rest3/binary>>) -> + {{<<"type">>, <<"chat">>}, Rest3}; + (<<12:8, Rest3/binary>>) -> + {{<<"type">>, <<"groupchat">>}, Rest3}; + (<<13:8, Rest3/binary>>) -> + {{<<"type">>, <<"normal">>}, Rest3}; + (<<14:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"type">>, AVal}, Rest4}; + (<<15:8, Rest3/binary>>) -> + {{<<"xml:lang">>, <<"en">>}, Rest3}; + (<<16:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"xml:lang">>, AVal}, Rest4}; + (<<2:8, Rest3/binary>>) -> + {stop, Rest3}; + (Data) -> + decode_attr(Data) + end, Rest), + {Children, Rest6} = decode_children(Rest2, Ns, J1, J2), + {{xmlel, <<"message">>, add_ns(PNs, Ns, Attrs), Children}, Rest6}; +decode(<<8:8, Rest/binary>>, PNs, J1, J2) -> + Ns = <<"jabber:client">>, + {Attrs, Rest2} = decode_attrs(Rest), + {Children, Rest6} = prefix_map(fun (<<9:8, Rest5/binary>>) -> + {{xmlcdata, <<73,32,115,101,110,116,32,121,111,117,32,97,110,32,79,77,69, + 77,79,32,101,110,99,114,121,112,116,101,100,32,109,101,115, + 115,97,103,101,32,98,117,116,32,121,111,117,114,32,99,108, + 105,101,110,116,32,100,111,101,115,110,226,128,153,116,32, + 115,101,101,109,32,116,111,32,115,117,112,112,111,114,116, + 32,116,104,97,116,46,32,70,105,110,100,32,109,111,114,101, + 32,105,110,102,111,114,109,97,116,105,111,110,32,111,110, + 32,104,116,116,112,115,58,47,47,99,111,110,118,101,114,115, + 97,116,105,111,110,115,46,105,109,47,111,109,101,109,111>>}, Rest5}; + (Other) -> + decode_child(Other, Ns, J1, J2) + end, Rest2), + {{xmlel, <<"body">>, add_ns(PNs, Ns, Attrs), Children}, Rest6}; +decode(<<31:8, Rest/binary>>, PNs, J1, J2) -> + Ns = <<"jabber:client">>, + {Attrs, Rest2} = decode_attrs(Rest), + {Children, Rest6} = decode_children(Rest2, Ns, J1, J2), + {{xmlel, <<"subject">>, add_ns(PNs, Ns, Attrs), Children}, Rest6}; +decode(<<32:8, Rest/binary>>, PNs, J1, J2) -> + Ns = <<"jabber:client">>, + {Attrs, Rest2} = decode_attrs(Rest), + {Children, Rest6} = decode_children(Rest2, Ns, J1, J2), + {{xmlel, <<"thread">>, add_ns(PNs, Ns, Attrs), Children}, Rest6}; +decode(<<7:8, Rest/binary>>, PNs, J1, J2) -> + Ns = <<"urn:xmpp:hints">>, + {Attrs, Rest2} = decode_attrs(Rest), + {Children, Rest6} = decode_children(Rest2, Ns, J1, J2), + {{xmlel, <<"store">>, add_ns(PNs, Ns, Attrs), Children}, Rest6}; +decode(<<10:8, Rest/binary>>, PNs, J1, J2) -> + Ns = <<"urn:xmpp:sid:0">>, + {Attrs, Rest2} = prefix_map(fun + (<<3:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"id">>, AVal}, Rest4}; + (<<2:8, Rest3/binary>>) -> + {stop, Rest3}; + (Data) -> + decode_attr(Data) + end, Rest), + {Children, Rest6} = decode_children(Rest2, Ns, J1, J2), + {{xmlel, <<"origin-id">>, add_ns(PNs, Ns, Attrs), Children}, Rest6}; +decode(<<22:8, Rest/binary>>, PNs, J1, J2) -> + Ns = <<"urn:xmpp:sid:0">>, + {Attrs, Rest2} = prefix_map(fun + (<<3:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"by">>, AVal}, Rest4}; + (<<4:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"id">>, AVal}, Rest4}; + (<<2:8, Rest3/binary>>) -> + {stop, Rest3}; + (Data) -> + decode_attr(Data) + end, Rest), + {Children, Rest6} = decode_children(Rest2, Ns, J1, J2), + {{xmlel, <<"stanza-id">>, add_ns(PNs, Ns, Attrs), Children}, Rest6}; +decode(<<11:8, Rest/binary>>, PNs, J1, J2) -> + Ns = <<"urn:xmpp:chat-markers:0">>, + {Attrs, Rest2} = decode_attrs(Rest), + {Children, Rest6} = decode_children(Rest2, Ns, J1, J2), + {{xmlel, <<"markable">>, add_ns(PNs, Ns, Attrs), Children}, Rest6}; +decode(<<20:8, Rest/binary>>, PNs, J1, J2) -> + Ns = <<"urn:xmpp:chat-markers:0">>, + {Attrs, Rest2} = prefix_map(fun + (<<3:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"id">>, AVal}, Rest4}; + (<<4:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"sender">>, <>}, Rest4}; + (<<5:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"sender">>, <>}, Rest4}; + (<<6:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"sender">>, AVal}, Rest4}; + (<<2:8, Rest3/binary>>) -> + {stop, Rest3}; + (Data) -> + decode_attr(Data) + end, Rest), + {Children, Rest6} = decode_children(Rest2, Ns, J1, J2), + {{xmlel, <<"displayed">>, add_ns(PNs, Ns, Attrs), Children}, Rest6}; +decode(<<24:8, Rest/binary>>, PNs, J1, J2) -> + Ns = <<"urn:xmpp:chat-markers:0">>, + {Attrs, Rest2} = prefix_map(fun + (<<3:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"id">>, AVal}, Rest4}; + (<<2:8, Rest3/binary>>) -> + {stop, Rest3}; + (Data) -> + decode_attr(Data) + end, Rest), + {Children, Rest6} = decode_children(Rest2, Ns, J1, J2), + {{xmlel, <<"received">>, add_ns(PNs, Ns, Attrs), Children}, Rest6}; +decode(<<16:8, Rest/binary>>, PNs, J1, J2) -> + Ns = <<"urn:xmpp:eme:0">>, + {Attrs, Rest2} = prefix_map(fun + (<<3:8, Rest3/binary>>) -> + {{<<"name">>, <<"OMEMO">>}, Rest3}; + (<<4:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"name">>, AVal}, Rest4}; + (<<5:8, Rest3/binary>>) -> + {{<<"namespace">>, <<"eu.siacs.conversations.axolotl">>}, Rest3}; + (<<6:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"namespace">>, AVal}, Rest4}; + (<<2:8, Rest3/binary>>) -> + {stop, Rest3}; + (Data) -> + decode_attr(Data) + end, Rest), + {Children, Rest6} = decode_children(Rest2, Ns, J1, J2), + {{xmlel, <<"encryption">>, add_ns(PNs, Ns, Attrs), Children}, Rest6}; +decode(<<17:8, Rest/binary>>, PNs, J1, J2) -> + Ns = <<"urn:xmpp:delay">>, + {Attrs, Rest2} = prefix_map(fun + (<<3:8, Rest3/binary>>) -> + {{<<"from">>, J1}, Rest3}; + (<<4:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"from">>, AVal}, Rest4}; + (<<5:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"stamp">>, AVal}, Rest4}; + (<<2:8, Rest3/binary>>) -> + {stop, Rest3}; + (Data) -> + decode_attr(Data) + end, Rest), + {Children, Rest6} = decode_children(Rest2, Ns, J1, J2), + {{xmlel, <<"delay">>, add_ns(PNs, Ns, Attrs), Children}, Rest6}; +decode(<<18:8, Rest/binary>>, PNs, J1, J2) -> + Ns = <<"http://jabber.org/protocol/address">>, + {Attrs, Rest2} = prefix_map(fun + (<<3:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"jid">>, <>}, Rest4}; + (<<4:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"jid">>, AVal}, Rest4}; + (<<5:8, Rest3/binary>>) -> + {{<<"type">>, <<"ofrom">>}, Rest3}; + (<<6:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"type">>, AVal}, Rest4}; + (<<2:8, Rest3/binary>>) -> + {stop, Rest3}; + (Data) -> + decode_attr(Data) + end, Rest), + {Children, Rest6} = decode_children(Rest2, Ns, J1, J2), + {{xmlel, <<"address">>, add_ns(PNs, Ns, Attrs), Children}, Rest6}; +decode(<<19:8, Rest/binary>>, PNs, J1, J2) -> + Ns = <<"http://jabber.org/protocol/address">>, + {Attrs, Rest2} = decode_attrs(Rest), + {Children, Rest6} = decode_children(Rest2, Ns, J1, J2), + {{xmlel, <<"addresses">>, add_ns(PNs, Ns, Attrs), Children}, Rest6}; +decode(<<21:8, Rest/binary>>, PNs, J1, J2) -> + Ns = <<"urn:xmpp:mam:tmp">>, + {Attrs, Rest2} = prefix_map(fun + (<<3:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"by">>, AVal}, Rest4}; + (<<4:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"id">>, AVal}, Rest4}; + (<<2:8, Rest3/binary>>) -> + {stop, Rest3}; + (Data) -> + decode_attr(Data) + end, Rest), + {Children, Rest6} = decode_children(Rest2, Ns, J1, J2), + {{xmlel, <<"archived">>, add_ns(PNs, Ns, Attrs), Children}, Rest6}; +decode(<<23:8, Rest/binary>>, PNs, J1, J2) -> + Ns = <<"urn:xmpp:receipts">>, + {Attrs, Rest2} = decode_attrs(Rest), + {Children, Rest6} = decode_children(Rest2, Ns, J1, J2), + {{xmlel, <<"request">>, add_ns(PNs, Ns, Attrs), Children}, Rest6}; +decode(<<25:8, Rest/binary>>, PNs, J1, J2) -> + Ns = <<"urn:xmpp:receipts">>, + {Attrs, Rest2} = prefix_map(fun + (<<3:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"id">>, AVal}, Rest4}; + (<<2:8, Rest3/binary>>) -> + {stop, Rest3}; + (Data) -> + decode_attr(Data) + end, Rest), + {Children, Rest6} = decode_children(Rest2, Ns, J1, J2), + {{xmlel, <<"received">>, add_ns(PNs, Ns, Attrs), Children}, Rest6}; +decode(<<26:8, Rest/binary>>, PNs, J1, J2) -> + Ns = <<"http://jabber.org/protocol/chatstates">>, + {Attrs, Rest2} = decode_attrs(Rest), + {Children, Rest6} = decode_children(Rest2, Ns, J1, J2), + {{xmlel, <<"active">>, add_ns(PNs, Ns, Attrs), Children}, Rest6}; +decode(<<39:8, Rest/binary>>, PNs, J1, J2) -> + Ns = <<"http://jabber.org/protocol/chatstates">>, + {Attrs, Rest2} = decode_attrs(Rest), + {Children, Rest6} = decode_children(Rest2, Ns, J1, J2), + {{xmlel, <<"composing">>, add_ns(PNs, Ns, Attrs), Children}, Rest6}; +decode(<<27:8, Rest/binary>>, PNs, J1, J2) -> + Ns = <<"http://jabber.org/protocol/muc#user">>, + {Attrs, Rest2} = prefix_map(fun + (<<3:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"from">>, <>}, Rest4}; + (<<4:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"from">>, AVal}, Rest4}; + (<<2:8, Rest3/binary>>) -> + {stop, Rest3}; + (Data) -> + decode_attr(Data) + end, Rest), + {Children, Rest6} = decode_children(Rest2, Ns, J1, J2), + {{xmlel, <<"invite">>, add_ns(PNs, Ns, Attrs), Children}, Rest6}; +decode(<<28:8, Rest/binary>>, PNs, J1, J2) -> + Ns = <<"http://jabber.org/protocol/muc#user">>, + {Attrs, Rest2} = decode_attrs(Rest), + {Children, Rest6} = decode_children(Rest2, Ns, J1, J2), + {{xmlel, <<"reason">>, add_ns(PNs, Ns, Attrs), Children}, Rest6}; +decode(<<29:8, Rest/binary>>, PNs, J1, J2) -> + Ns = <<"http://jabber.org/protocol/muc#user">>, + {Attrs, Rest2} = decode_attrs(Rest), + {Children, Rest6} = decode_children(Rest2, Ns, J1, J2), + {{xmlel, <<"x">>, add_ns(PNs, Ns, Attrs), Children}, Rest6}; +decode(<<30:8, Rest/binary>>, PNs, J1, J2) -> + Ns = <<"jabber:x:conference">>, + {Attrs, Rest2} = prefix_map(fun + (<<3:8, Rest3/binary>>) -> + {{<<"jid">>, J2}, Rest3}; + (<<4:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"jid">>, AVal}, Rest4}; + (<<2:8, Rest3/binary>>) -> + {stop, Rest3}; + (Data) -> + decode_attr(Data) + end, Rest), + {Children, Rest6} = decode_children(Rest2, Ns, J1, J2), + {{xmlel, <<"x">>, add_ns(PNs, Ns, Attrs), Children}, Rest6}; +decode(<<33:8, Rest/binary>>, PNs, J1, J2) -> + Ns = <<"http://jabber.org/protocol/pubsub#event">>, + {Attrs, Rest2} = decode_attrs(Rest), + {Children, Rest6} = decode_children(Rest2, Ns, J1, J2), + {{xmlel, <<"event">>, add_ns(PNs, Ns, Attrs), Children}, Rest6}; +decode(<<34:8, Rest/binary>>, PNs, J1, J2) -> + Ns = <<"http://jabber.org/protocol/pubsub#event">>, + {Attrs, Rest2} = prefix_map(fun + (<<3:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"id">>, AVal}, Rest4}; + (<<2:8, Rest3/binary>>) -> + {stop, Rest3}; + (Data) -> + decode_attr(Data) + end, Rest), + {Children, Rest6} = decode_children(Rest2, Ns, J1, J2), + {{xmlel, <<"item">>, add_ns(PNs, Ns, Attrs), Children}, Rest6}; +decode(<<35:8, Rest/binary>>, PNs, J1, J2) -> + Ns = <<"http://jabber.org/protocol/pubsub#event">>, + {Attrs, Rest2} = prefix_map(fun + (<<3:8, Rest3/binary>>) -> + {{<<"node">>, <<"urn:xmpp:mucsub:nodes:messages">>}, Rest3}; + (<<4:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"node">>, AVal}, Rest4}; + (<<2:8, Rest3/binary>>) -> + {stop, Rest3}; + (Data) -> + decode_attr(Data) + end, Rest), + {Children, Rest6} = decode_children(Rest2, Ns, J1, J2), + {{xmlel, <<"items">>, add_ns(PNs, Ns, Attrs), Children}, Rest6}; +decode(<<36:8, Rest/binary>>, PNs, J1, J2) -> + Ns = <<"p1:push:custom">>, + {Attrs, Rest2} = prefix_map(fun + (<<3:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"key">>, AVal}, Rest4}; + (<<4:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"value">>, AVal}, Rest4}; + (<<2:8, Rest3/binary>>) -> + {stop, Rest3}; + (Data) -> + decode_attr(Data) + end, Rest), + {Children, Rest6} = decode_children(Rest2, Ns, J1, J2), + {{xmlel, <<"x">>, add_ns(PNs, Ns, Attrs), Children}, Rest6}; +decode(<<37:8, Rest/binary>>, PNs, J1, J2) -> + Ns = <<"p1:pushed">>, + {Attrs, Rest2} = decode_attrs(Rest), + {Children, Rest6} = decode_children(Rest2, Ns, J1, J2), + {{xmlel, <<"x">>, add_ns(PNs, Ns, Attrs), Children}, Rest6}; +decode(<<38:8, Rest/binary>>, PNs, J1, J2) -> + Ns = <<"urn:xmpp:message-correct:0">>, + {Attrs, Rest2} = prefix_map(fun + (<<3:8, Rest3/binary>>) -> + {AVal, Rest4} = decode_string(Rest3), + {{<<"id">>, AVal}, Rest4}; + (<<2:8, Rest3/binary>>) -> + {stop, Rest3}; + (Data) -> + decode_attr(Data) + end, Rest), + {Children, Rest6} = decode_children(Rest2, Ns, J1, J2), + {{xmlel, <<"replace">>, add_ns(PNs, Ns, Attrs), Children}, Rest6}; +decode(Other, PNs, J1, J2) -> + decode_child(Other, PNs, J1, J2). + diff --git a/tools/xml_compress_gen.erl b/tools/xml_compress_gen.erl new file mode 100644 index 000000000..4dad71a43 --- /dev/null +++ b/tools/xml_compress_gen.erl @@ -0,0 +1,417 @@ +%% File : xml_compress_gen.erl +%% Author : Pawel Chmielowski +%% Purpose : +%% Created : 14 Sep 2018 Pawel Chmielowski +%% +%% +%% ejabberd, Copyright (C) 2002-2018 ProcessOne +%% +%% This program is free software; you can redistribute it and/or +%% modify it under the terms of the GNU General Public License as +%% published by the Free Software Foundation; either version 2 of the +%% License, or (at your option) any later version. +%% +%% This program is distributed in the hope that it will be useful, +%% but WITHOUT ANY WARRANTY; without even the implied warranty of +%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +%% General Public License for more details. +%% +%% You should have received a copy of the GNU General Public License along +%% with this program; if not, write to the Free Software Foundation, Inc., +%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +%% + +-module(xml_compress_gen). +-author("pawel@process-one.net"). + +-include("xmpp.hrl"). + +%% API +-export([archive_analyze/3, process_stats/1, gen_code/3]). + +-record(el_stats, {count = 0, empty_count = 0, only_text_count = 0, attrs = #{}, text_stats = #{}}). +-record(attr_stats, {count = 0, vals = #{}}). + +archive_analyze(Host, Table, EHost) -> + case ejabberd_sql:sql_query(Host, <<"select username, peer, kind, xml from ", Table/binary>>) of + {selected, _, Res} -> + lists:foldl( + fun([U, P, K, X], Stats) -> + M = case K of + <<"groupchat">> -> + U; + _ -> + <> + end, + El = fxml_stream:parse_element(X), + analyze_element({El, <<"stream">>, <<"jabber:client">>, M, P}, Stats) + end, {0, #{}}, Res); + _ -> + none + end. + +encode_id(Num) when Num < 64 -> + iolist_to_binary(io_lib:format("~p:8", [Num])). + +gen_code(_File, _Rules, $<) -> + {error, <<"Invalid version">>}; +gen_code(File, Rules, Ver) when Ver < 64 -> + {Data, _} = lists:foldl( + fun({Ns, El, Attrs, Text}, {Acc, Id}) -> + NsC = case lists:keyfind(Ns, 1, Acc) of + false -> []; + {_, L} -> L + end, + {AttrsE, _} = lists:mapfoldl( + fun({AName, AVals}, Id2) -> + {AD, Id3} = lists:mapfoldl( + fun(AVal, Id3) -> + {{AVal, encode_id(Id3)}, Id3 + 1} + end, Id2, AVals), + {{AName, AD ++ [encode_id(Id3)]}, Id3 + 1} + end, 3, Attrs), + {TextE, Id5} = lists:mapfoldl( + fun(TextV, Id4) -> + {{TextV, encode_id(Id4)}, Id4 + 1} + end, Id + 1, Text), + {lists:keystore(Ns, 1, Acc, {Ns, NsC ++ [{El, encode_id(Id), AttrsE, TextE}]}), Id5} + end, {[], 5}, Rules), + {ok, Dev} = file:open(File, write), + Mod = filename:basename(File, ".erl"), + io:format(Dev, "-module(~s).~n-export([encode/3, decode/3]).~n~n", [Mod]), + RulesS = iolist_to_binary(io_lib:format("~p", [Rules])), + RulesS2 = binary:replace(RulesS, <<"\n">>, <<"\n% ">>, [global]), + io:format(Dev, "% This file was generated by xml_compress_gen~n%~n" + "% Rules used:~n%~n% ~s~n~n", [RulesS2]), + VerId = iolist_to_binary(io_lib:format("~p:8", [Ver])), + gen_encode(Dev, Data, VerId), + gen_decode(Dev, Data, VerId), + file:close(Dev), + Data. + +gen_decode(Dev, Data, VerId) -> + io:format(Dev, "decode(<<$<, _/binary>> = Data, _J1, _J2) ->~n" + " fxml_stream:parse_element(Data);~n" + "decode(<<~s, Rest/binary>>, J1, J2) ->~n" + " {El, _} = decode(Rest, <<\"jabber:client\">>, J1, J2),~n" + " El.~n~n", [VerId]), + io:format(Dev, "decode_string(Data) ->~n" + " case Data of~n" + " <<0:2, L:6, Str:L/binary, Rest/binary>> ->~n" + " {Str, Rest};~n" + " <<1:2, L1:6, 0:2, L2:6, Rest/binary>> ->~n" + " L = L2*64 + L1,~n" + " <> = Rest,~n" + " {Str, Rest2};~n" + " <<1:2, L1:6, 1:2, L2:6, L3:8, Rest/binary>> ->~n" + " L = (L3*64 + L2)*64 + L1,~n" + " <> = Rest,~n" + " {Str, Rest2}~n" + " end.~n~n", []), + io:format(Dev, "decode_child(<<1:8, Rest/binary>>, _PNs, _J1, _J2) ->~n" + " {Text, Rest2} = decode_string(Rest),~n" + " {{xmlcdata, Text}, Rest2};~n", []), + io:format(Dev, "decode_child(<<2:8, Rest/binary>>, PNs, J1, J2) ->~n" + " {Name, Rest2} = decode_string(Rest),~n" + " {Attrs, Rest3} = decode_attrs(Rest2),~n" + " {Children, Rest4} = decode_children(Rest3, PNs, J1, J2),~n" + " {{xmlel, Name, Attrs, Children}, Rest4};~n", []), + io:format(Dev, "decode_child(<<3:8, Rest/binary>>, PNs, J1, J2) ->~n" + " {Name, Rest2} = decode_string(Rest),~n" + " {Ns, Rest3} = decode_string(Rest2),~n" + " {Attrs, Rest4} = decode_attrs(Rest3),~n" + " {Children, Rest5} = decode_children(Rest4, Ns, J1, J2),~n" + " {{xmlel, Name, add_ns(PNs, Ns, Attrs), Children}, Rest5};~n", []), + io:format(Dev, "decode_child(<<4:8, Rest/binary>>, _PNs, _J1, _J2) ->~n" + " {stop, Rest};~n", []), + io:format(Dev, "decode_child(Other, PNs, J1, J2) ->~n" + " decode(Other, PNs, J1, J2).~n~n", []), + io:format(Dev, "decode_children(Data, PNs, J1, J2) ->~n" + " prefix_map(fun(Data2) -> decode(Data2, PNs, J1, J2) end, Data).~n~n", []), + io:format(Dev, "decode_attr(<<1:8, Rest/binary>>) ->~n" + " {Name, Rest2} = decode_string(Rest),~n" + " {Val, Rest3} = decode_string(Rest2),~n" + " {{Name, Val}, Rest3};~n", []), + io:format(Dev, "decode_attr(<<2:8, Rest/binary>>) ->~n" + " {stop, Rest}.~n~n", []), + io:format(Dev, "decode_attrs(Data) ->~n" + " prefix_map(fun decode_attr/1, Data).~n~n", []), + io:format(Dev, "prefix_map(F, Data) ->~n" + " prefix_map(F, Data, []).~n~n", []), + io:format(Dev, "prefix_map(F, Data, Acc) ->~n" + " case F(Data) of~n" + " {stop, Rest} ->~n" + " {lists:reverse(Acc), Rest};~n" + " {Val, Rest} ->~n" + " prefix_map(F, Rest, [Val | Acc])~n" + " end.~n~n", []), + io:format(Dev, "add_ns(Ns, Ns, Attrs) ->~n" + " Attrs;~n" + "add_ns(_, Ns, Attrs) ->~n" + " [{<<\"xmlns\">>, Ns} | Attrs].~n~n", []), + lists:foreach( + fun({Ns, Els}) -> + lists:foreach( + fun({Name, Id, Attrs, Text}) -> + io:format(Dev, "decode(<<~s, Rest/binary>>, PNs, J1, J2) ->~n" + " Ns = ~p,~n", [Id, Ns]), + case Attrs of + [] -> + io:format(Dev, " {Attrs, Rest2} = decode_attrs(Rest),~n", []); + _ -> + io:format(Dev, " {Attrs, Rest2} = prefix_map(fun~n", []), + lists:foreach( + fun({AName, AVals}) -> + lists:foreach( + fun({j1, AId}) -> + io:format(Dev, " (<<~s, Rest3/binary>>) ->~n" + " {{~p, J1}, Rest3};~n", [AId, AName]); + ({j2, AId}) -> + io:format(Dev, " (<<~s, Rest3/binary>>) ->~n" + " {{~p, J2}, Rest3};~n", [AId, AName]); + ({{j1}, AId}) -> + io:format(Dev, " (<<~s, Rest3/binary>>) ->~n" + " {AVal, Rest4} = decode_string(Rest3),~n" + " {{~p, <>}, Rest4};~n", + [AId, AName]); + ({{j2}, AId}) -> + io:format(Dev, " (<<~s, Rest3/binary>>) ->~n" + " {AVal, Rest4} = decode_string(Rest3),~n" + " {{~p, <>}, Rest4};~n", + [AId, AName]); + ({AVal, AId}) -> + io:format(Dev, " (<<~s, Rest3/binary>>) ->~n" + " {{~p, ~p}, Rest3};~n", + [AId, AName, AVal]); + (AId) -> + io:format(Dev, " (<<~s, Rest3/binary>>) ->~n" + " {AVal, Rest4} = decode_string(Rest3),~n" + " {{~p, AVal}, Rest4};~n", + [AId, AName]) + end, AVals) + end, Attrs), + io:format(Dev, " (<<2:8, Rest3/binary>>) ->~n" + " {stop, Rest3};~n" + " (Data) ->~n" + " decode_attr(Data)~n" + " end, Rest),~n", []) + end, + case Text of + [] -> + io:format(Dev, " {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),~n", []); + _ -> + io:format(Dev, " {Children, Rest6} = prefix_map(fun", []), + lists:foreach( + fun({TextS, TId}) -> + io:format(Dev, " (<<~s, Rest5/binary>>) ->~n" + " {{xmlcdata, ~p}, Rest5};~n", + [TId, TextS]) + end, Text), + + io:format(Dev, " (Other) ->~n" + " decode_child(Other, Ns, J1, J2)~n" + " end, Rest2),~n", []) + end, + io:format(Dev, " {{xmlel, ~p, add_ns(PNs, Ns, Attrs), Children}, Rest6};~n", [Name]) + end, Els) + end, Data), + io:format(Dev, "decode(Other, PNs, J1, J2) ->~n" + " decode_child(Other, PNs, J1, J2).~n~n", []). + + +gen_encode(Dev, Data, VerId) -> + io:format(Dev, "encode(El, J1, J2) ->~n" + " encode_child(El, <<\"jabber:client\">>,~n" + " J1, J2, byte_size(J1), byte_size(J2), <<~s>>).~n~n", [VerId]), + io:format(Dev, "encode_attr({<<\"xmlns\">>, _}, Acc) ->~n" + " Acc;~n" + "encode_attr({N, V}, Acc) ->~n" + " <>.~n~n", []), + io:format(Dev, "encode_attrs(Attrs, Acc) ->~n" + " lists:foldl(fun encode_attr/2, Acc, Attrs).~n~n", []), + io:format(Dev, "encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) ->~n" + " E1 = if~n" + " PNs == Ns -> encode_attrs(Attrs, <>);~n" + " true -> encode_attrs(Attrs, <>)~n" + " end,~n" + " E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <>),~n" + " <>.~n~n", []), + io:format(Dev, "encode_child({xmlel, Name, Attrs, Children}, PNs, J1, J2, J1L, J2L, Pfx) ->~n" + " case lists:keyfind(<<\"xmlns\">>, 1, Attrs) of~n" + " false ->~n" + " encode(PNs, PNs, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx);~n" + " {_, Ns} ->~n" + " encode(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx)~n" + " end;~n" + "encode_child({xmlcdata, Data}, _PNs, _J1, _J2, _J1L, _J2L, Pfx) ->~n" + " <>.~n~n", []), + io:format(Dev, "encode_children(Children, PNs, J1, J2, J1L, J2L, Pfx) ->~n" + " lists:foldl(~n" + " fun(Child, Acc) ->~n" + " encode_child(Child, PNs, J1, J2, J1L, J2L, Acc)~n" + " end, Pfx, Children).~n~n", []), + io:format(Dev, "encode_string(Data) ->~n" + " <> = <<(byte_size(Data)):16/unsigned-big-integer>>,~n" + " case {V1, V2, V3} of~n" + " {0, 0, V3} ->~n" + " <>;~n" + " {0, V2, V3} ->~n" + " <<(V3 bor 64):8, V2:8, Data/binary>>;~n" + " _ ->~n" + " <<(V3 bor 64):8, (V2 bor 64):8, V1:8, Data/binary>>~n" + " end.~n~n", []), + lists:foreach( + fun({Ns, Els}) -> + io:format(Dev, "encode(PNs, ~p = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) ->~n" + " case Name of~n", [Ns]), + lists:foreach( + fun({ElN, Id, Attrs, Text}) -> + io:format(Dev, " ~p ->~n", [ElN]), + case Attrs of + [] -> + io:format(Dev, " E = encode_attrs(Attrs, <>),~n", [Id]); + _ -> + io:format(Dev, " E = lists:foldl(fun~n", []), + lists:foreach( + fun({AName, AVals}) -> + case AVals of + [AIdS] when is_binary(AIdS) -> + io:format(Dev, " ({~p, AVal}, Acc) ->~n" + " <>;~n", + [AName, AIdS]); + _ -> + io:format(Dev, " ({~p, AVal}, Acc) ->~n" + " case AVal of~n", [AName]), + lists:foreach( + fun({j1, AId}) -> + io:format(Dev, " J1 -> <>;~n", + [AId]); + ({j2, AId}) -> + io:format(Dev, " J2 -> <>;~n", + [AId]); + ({{j1}, AId}) -> + io:format(Dev, " <> -> " + "<>;~n", + [AId]); + ({{j2}, AId}) -> + io:format(Dev, " <> -> " + "<>;~n", + [AId]); + ({AVal, AId}) -> + io:format(Dev, " ~p -> <>;~n", + [AVal, AId]); + (AId) -> + io:format(Dev, " _ -> <>~n", + [AId]) + end, AVals), + io:format(Dev, " end;~n", []) + end + end, Attrs), + io:format(Dev, " (Attr, Acc) -> encode_attr(Attr, Acc)~n", []), + io:format(Dev, " end, <>, Attrs),~n", [Id]) + end, + case Text of + [] -> + io:format(Dev, " E2 = encode_children(Children, Ns, " + "J1, J2, J1L, J2L, <>),~n", []); + _ -> + io:format(Dev, " E2 = lists:foldl(fun~n", []), + lists:foreach( + fun({TextV, TId}) -> + io:format(Dev, " ({xmlcdata, ~p}, Acc) -> <>;~n", [TextV, TId]) + end, Text), + io:format(Dev, " (El, Acc) -> encode_child(El, Ns, J1, J2, J1L, J2L, Acc)~n", []), + io:format(Dev, " end, <>, Children),~n", []) + end, + io:format(Dev, " <>;~n", []) + end, Els), + io:format(Dev, " _ -> encode_el(PNs, Ns, Name, Attrs, Children, " + "J1, J2, J1L, J2L, Pfx)~nend;~n", []) + end, Data), + io:format(Dev, "encode(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) ->~n" + " encode_el(PNs, Ns, Name, Attrs, Children, " + "J1, J2, J1L, J2L, Pfx).~n~n", []). + +process_stats({_Counts, Stats}) -> + SStats = lists:sort( + fun({_, #el_stats{count = C1}}, {_, #el_stats{count = C2}}) -> + C1 >= C2 + end, maps:to_list(Stats)), + lists:map( + fun({Name, #el_stats{count = C, attrs = A, text_stats = T}}) -> + [Ns, El] = binary:split(Name, <<"<">>), + Attrs = lists:filtermap( + fun({AN, #attr_stats{count = AC, vals = AV}}) -> + if + AC*5 < C -> + false; + true -> + AVC = AC div min(maps:size(AV)*2, 10), + AVA = [N || {N, C2} <- maps:to_list(AV), C2 > AVC], + {true, {AN, AVA}} + end + end, maps:to_list(A)), + Text = [TE || {TE, TC} <- maps:to_list(T), TC > C/2], + {Ns, El, Attrs, Text} + end, SStats). + +analyze_elements(Elements, Stats, PName, PNS, J1, J2) -> + lists:foldl(fun analyze_element/2, Stats, lists:map(fun(V) -> {V, PName, PNS, J1, J2} end, Elements)). + +maps_update(Key, F, InitVal, Map) -> + case maps:is_key(Key, Map) of + true -> + maps:update_with(Key, F, Map); + _ -> + maps:put(Key, F(InitVal), Map) + end. + +analyze_element({{xmlcdata, Data}, PName, PNS, _J1, _J2}, {ElCount, Stats}) -> + Stats2 = maps_update(<>, + fun(#el_stats{text_stats = TS} = E) -> + TS2 = maps_update(Data, fun(C) -> C + 1 end, 0, TS), + E#el_stats{text_stats = TS2} + end, #el_stats{}, Stats), + {ElCount, Stats2}; +analyze_element({#xmlel{name = Name, attrs = Attrs, children = Children}, _PName, PNS, J1, J2}, {ElCount, Stats}) -> + XMLNS = case lists:keyfind(<<"xmlns">>, 1, Attrs) of + {_, NS} -> + NS; + false -> + PNS + end, + NStats = maps_update(<>, + fun(#el_stats{count = C, empty_count = EC, only_text_count = TC, attrs = A} = ES) -> + A2 = lists:foldl( + fun({<<"xmlns">>, _}, AMap) -> + AMap; + ({AName, AVal}, AMap) -> + J1S = size(J1), + J2S = size(J2), + AVal2 = case AVal of + J1 -> + j1; + J2 -> + j2; + <> -> + {j1}; + <> -> + {j2}; + Other -> + Other + end, + maps_update(AName, fun(#attr_stats{count = AC, vals = AV}) -> + AV2 = maps_update(AVal2, fun(C2) -> C2 + 1 end, 0, AV), + #attr_stats{count = AC + 1, vals = AV2} + end, #attr_stats{}, AMap) + end, A, Attrs), + ES#el_stats{count = C + 1, + empty_count = if Children == [] -> EC + 1; true -> + EC end, + only_text_count = case Children of [{xmlcdata, _}] -> TC + 1; _ -> TC end, + attrs = A2} + end, #el_stats{}, Stats), + analyze_elements(Children, {ElCount + 1, NStats}, Name, XMLNS, J1, J2). -- 2.40.0