From a7db8aa66ec26e6c459c0ee043e4f8f121a78a72 Mon Sep 17 00:00:00 2001 From: Aki Tuomi Date: Thu, 28 Nov 2013 15:14:29 +0200 Subject: [PATCH] Enable zeromq for remotebackend --- .travis.yml | 13 +- configure.ac | 15 ++ modules/remotebackend/.gitignore | 1 + modules/remotebackend/Gemfile | 2 + modules/remotebackend/Gemfile.lock | 8 ++ modules/remotebackend/Makefile.am | 54 ++++---- modules/remotebackend/OBJECTFILES | 2 +- modules/remotebackend/OBJECTLIBS | 2 +- modules/remotebackend/example.rb | 1 + .../remotebackend/regression-tests/Gemfile | 1 + .../regression-tests/Gemfile.lock | 8 +- .../remotebackend/regression-tests/backend.rb | 3 +- .../regression-tests/http-backend.rb | 4 +- .../regression-tests/pipe-backend.rb | 5 +- .../regression-tests/zeromq-backend.rb | 48 +++++++ modules/remotebackend/remotebackend.cc | 6 + modules/remotebackend/remotebackend.hh | 22 ++- .../test-remotebackend-zeromq.cc | 77 ++++++++++ modules/remotebackend/testrunner.sh | 32 +++++ modules/remotebackend/unittest.rb | 1 + modules/remotebackend/unittest_http.rb | 7 +- modules/remotebackend/unittest_json.rb | 7 +- modules/remotebackend/unittest_pipe.rb | 1 + modules/remotebackend/unittest_post.rb | 7 +- modules/remotebackend/unittest_zeromq.rb | 50 +++++++ modules/remotebackend/zmqconnector.cc | 131 ++++++++++++++++++ regression-tests/start-test-stop | 9 +- 27 files changed, 469 insertions(+), 48 deletions(-) create mode 100755 modules/remotebackend/regression-tests/zeromq-backend.rb create mode 100644 modules/remotebackend/test-remotebackend-zeromq.cc create mode 100755 modules/remotebackend/unittest_zeromq.rb create mode 100644 modules/remotebackend/zmqconnector.cc diff --git a/.travis.yml b/.travis.yml index 70a510a8f..7a51790a3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,20 +3,21 @@ compiler: - gcc - clang before_script: + - sudo rm /etc/apt/sources.list.d/travis_ci_zeromq3-source.list - sudo apt-get update - - sudo apt-get install libboost-all-dev libtolua-dev bc libcdb-dev libnet-dns-perl unbound-host ldnsutils dnsutils bind9utils libtool libcdb-dev xmlto dblatex links asciidoc ruby-json ruby-sqlite3 rubygems libcurl4-openssl-dev ruby1.9.1 socat time + - sudo apt-get install libboost-all-dev libtolua-dev bc libcdb-dev libnet-dns-perl unbound-host ldnsutils dnsutils bind9utils libtool libcdb-dev xmlto dblatex links asciidoc ruby-json ruby-sqlite3 rubygems libcurl4-openssl-dev ruby1.9.1 socat time libzmq1 libzmq-dev pkg-config - sudo update-alternatives --set ruby /usr/bin/ruby1.9.1 - - sudo gem install bundler --no-rdoc --no-ri --bindir /usr/local/bin + - gem install bundler --no-rdoc --no-ri - cd regression-tests - wget http://s3.amazonaws.com/alexa-static/top-1m.csv.zip - unzip top-1m.csv.zip - cd .. - cd modules/remotebackend - - sudo bundle install + - ruby -S bundle install - cd ../.. script: - ./bootstrap - - ./configure --with-modules='bind gmysql gpgsql gsqlite3 mydns tinydns remote random' --enable-unit-tests --enable-remotebackend-http --enable-tools + - ./configure --with-modules='bind gmysql gpgsql gsqlite3 mydns tinydns remote random' --enable-unit-tests --enable-remotebackend-http --enable-tools --enable-remotebackend-zeromq - make dist - make -j 4 - make -j 4 check @@ -49,6 +50,10 @@ script: - ./start-test-stop 5300 remotebackend-pipe-dnssec - ./start-test-stop 5300 remotebackend-unix - ./start-test-stop 5300 remotebackend-unix-dnssec + - ./start-test-stop 5300 remotebackend-http + - ./start-test-stop 5300 remotebackend-http-dnssec + - ./start-test-stop 5300 remotebackend-zeromq + - ./start-test-stop 5300 remotebackend-zeromq-dnssec - THRESHOLD=90 TRACE=no ./recursor-test 5300 - cd ../regression-tests.nobackend/ - ./runtests diff --git a/configure.ac b/configure.ac index 0e4dd1940..066fac415 100644 --- a/configure.ac +++ b/configure.ac @@ -282,6 +282,21 @@ then AC_SUBST(REMOTEBACKEND_HTTP) fi +AC_ARG_ENABLE(remotebackend_zeromq, AS_HELP_STRING([--enable-remotebackend-zeromq],[enable ZeroMQ connector for remotebackend]),[enable_remotebackend_zeromq=yes], [enable_remotebackend_zeromq=no]) +AC_MSG_CHECKING(whether to enable ZeroMQ connector in remotebackend) +AC_MSG_RESULT($enable_remotebackend_zeromq) +AM_CONDITIONAL(REMOTEBACKEND_HTTP,test x"$enable_remotebackend_zeromq" = "xyes") +if test "x$enable_remotebackend_zeromq" = "xyes" +then + PKG_CHECK_MODULES(LIBZMQ, libzmq, HAVE_LIBZMQ=yes, AC_MSG_ERROR([Could not find libzmq])) + REMOTEBACKEND_ZEROMQ=yes + AC_SUBST(LIBZMQ_LIBS) + AC_SUBST(LIBZMQ_CFLAGS) + AC_DEFINE(HAVE_LIBZMQ,1,[If we have libzmq]) + AC_DEFINE(REMOTEBACKEND_ZEROMQ,1,[If we want ZeroMQ connector]) + AC_SUBST(REMOTEBACKEND_ZEROMQ) +fi + AC_MSG_CHECKING(whether we should build static binaries) AC_ARG_ENABLE(static-binaries, AS_HELP_STRING([--enable-static-binaries],[Build static binaries]), diff --git a/modules/remotebackend/.gitignore b/modules/remotebackend/.gitignore index 572d12225..b040bc9e3 100644 --- a/modules/remotebackend/.gitignore +++ b/modules/remotebackend/.gitignore @@ -4,3 +4,4 @@ test_remotebackend_pipe test_remotebackend_unix test_remotebackend_json test_remotebackend_post +test_remotebackend_zeromq diff --git a/modules/remotebackend/Gemfile b/modules/remotebackend/Gemfile index 9eddca48f..70af3719c 100644 --- a/modules/remotebackend/Gemfile +++ b/modules/remotebackend/Gemfile @@ -1,3 +1,5 @@ source "https://rubygems.org" gem "webrick" +gem "zeromqrb" +gem "sqlite3" diff --git a/modules/remotebackend/Gemfile.lock b/modules/remotebackend/Gemfile.lock index b77c55326..44b2bee99 100644 --- a/modules/remotebackend/Gemfile.lock +++ b/modules/remotebackend/Gemfile.lock @@ -1,10 +1,18 @@ GEM remote: https://rubygems.org/ specs: + ffi (1.9.3) + ffi-rzmq (1.0.3) + ffi + sqlite3 (1.3.8) webrick (1.3.1) + zeromqrb (0.1.1) + ffi-rzmq (~> 1.0) PLATFORMS ruby DEPENDENCIES + sqlite3 webrick + zeromqrb diff --git a/modules/remotebackend/Makefile.am b/modules/remotebackend/Makefile.am index 7a2d79108..f7bf4b3d2 100644 --- a/modules/remotebackend/Makefile.am +++ b/modules/remotebackend/Makefile.am @@ -4,8 +4,8 @@ AM_CPPFLAGS=@THREADFLAGS@ $(BOOST_CPPFLAGS) $(LIBCURL_CFLAGS) -I../../pdns/ext/r # install .lib/libremotebackend.so.0.0.0 @libdir@ #endif -EXTRA_DIST=OBJECTFILES OBJECTLIBS testrunner.sh unittest_http.rb unittest_json.rb unittest_pipe.rb unittest_post.rb unittest.rb Gemfile Gemfile.lock -EXTRA_PROGRAMS=test_remotebackend_pipe test_remotebackend_unix test_remotebackend_http test_remotebackend_post test_remotebackend_json +EXTRA_DIST=OBJECTFILES OBJECTLIBS testrunner.sh unittest_http.rb unittest_json.rb unittest_pipe.rb unittest_zeromq.rb unittest_post.rb unittest.rb Gemfile Gemfile.lock +EXTRA_PROGRAMS=test_remotebackend_pipe test_remotebackend_unix test_remotebackend_http test_remotebackend_post test_remotebackend_json test_remotebackend_zeromq EXTRA_LTLIBRARIES=libtestremotebackend.la clean-local: @@ -13,13 +13,13 @@ clean-local: lib_LTLIBRARIES = libremotebackend.la -libremotebackend_la_SOURCES=remotebackend.hh remotebackend.cc unixconnector.cc httpconnector.cc pipeconnector.cc +libremotebackend_la_SOURCES=remotebackend.hh remotebackend.cc unixconnector.cc httpconnector.cc pipeconnector.cc zmqconnector.cc libremotebackend_la_LDFLAGS=-module -avoid-version -libremotebackend_la_LIBADD=$(LIBCURL_LIBS) +libremotebackend_la_LIBADD=$(LIBCURL_LIBS) $(LIBZMQ_LIBS) -TESTS_ENVIRONMENT = env BOOST_TEST_LOG_LEVEL=message REMOTEBACKEND_HTTP=$(REMOTEBACKEND_HTTP) ./testrunner.sh -TESTS=test_remotebackend_pipe test_remotebackend_unix test_remotebackend_http test_remotebackend_post test_remotebackend_json +TESTS_ENVIRONMENT = env BOOST_TEST_LOG_LEVEL=message REMOTEBACKEND_HTTP=$(REMOTEBACKEND_HTTP) REMOTEBACKEND_ZEROMQ==$(REMOTEBACKEND_ZEROMQ) ./testrunner.sh +TESTS=test_remotebackend_pipe test_remotebackend_unix test_remotebackend_http test_remotebackend_post test_remotebackend_json test_remotebackend_zeromq BUILT_SOURCES=../../pdns/dnslabeltext.cc @@ -35,33 +35,39 @@ libtestremotebackend_la_SOURCES=../../pdns/dnsbackend.hh ../../pdns/dnsbackend.c ../../pdns/aes/aescpp.h ../../pdns/dns.hh ../../pdns/dns.cc ../../pdns/json.hh ../../pdns/json.cc \ ../../pdns/aes/aescrypt.c ../../pdns/aes/aes.h ../../pdns/aes/aeskey.c ../../pdns/aes/aes_modes.c ../../pdns/aes/aesopt.h \ ../../pdns/aes/aestab.c ../../pdns/aes/aestab.h ../../pdns/aes/brg_endian.h ../../pdns/aes/brg_types.h \ - remotebackend.hh remotebackend.cc unixconnector.cc httpconnector.cc pipeconnector.cc + remotebackend.hh remotebackend.cc unixconnector.cc httpconnector.cc pipeconnector.cc zmqconnector.cc -libtestremotebackend_la_CFLAGS=$(BOOST_CPPFLAGS) @THREADFLAGS@ $(POLARSSL_CFLAGS) $(LIBCURL_CFLAGS) -g -O0 -I../../pdns -libtestremotebackend_la_CXXFLAGS=$(BOOST_CPPFLAGS) @THREADFLAGS@ $(POLARSSL_CFLAGS) $(LIBCURL_CFLAGS) -g -O0 -I../../pdns +libtestremotebackend_la_CFLAGS=$(BOOST_CPPFLAGS) @THREADFLAGS@ $(POLARSSL_CFLAGS) $(LIBCURL_CFLAGS) $(LIBZMQ_CFLAGS) -g -O0 -I../../pdns +libtestremotebackend_la_CXXFLAGS=$(BOOST_CPPFLAGS) @THREADFLAGS@ $(POLARSSL_CFLAGS) $(LIBCURL_CFLAGS) $(LIBZMQ_CFLAGS) -g -O0 -I../../pdns test_remotebackend_pipe_SOURCES=test-remotebackend.cc test-remotebackend-pipe.cc test-remotebackend-keys.hh test_remotebackend_unix_SOURCES=test-remotebackend.cc test-remotebackend-unix.cc test-remotebackend-keys.hh test_remotebackend_http_SOURCES=test-remotebackend.cc test-remotebackend-http.cc test-remotebackend-keys.hh test_remotebackend_post_SOURCES=test-remotebackend.cc test-remotebackend-post.cc test-remotebackend-keys.hh test_remotebackend_json_SOURCES=test-remotebackend.cc test-remotebackend-json.cc test-remotebackend-keys.hh +test_remotebackend_zeromq_SOURCES=test-remotebackend.cc test-remotebackend-zeromq.cc test-remotebackend-keys.hh -test_remotebackend_pipe_CFLAGS=$(BOOST_CPPFLAGS) @THREADFLAGS@ $(LIBCURL_CFLAGS) -g -O0 -I../../pdns -test_remotebackend_pipe_CXXFLAGS=$(BOOST_CPPFLAGS) @THREADFLAGS@ $(LIBCURL_CFLAGS) -g -O0 -I../../pdns -test_remotebackend_pipe_LDADD=libtestremotebackend.la @DYNLINKFLAGS@ @THREADFLAGS@ $(BOOST_UNIT_TEST_FRAMEWORK_LDFLAGS) $(BOOST_UNIT_TEST_FRAMEWORK_LIBS) $(BOOST_SERIALIZATION_LIBS) $(BOOST_PROGRAM_OPTIONS_LIBS) @LIBDL@ $(POLARSSL_LIBS) $(LIBCURL_LIBS) +test_remotebackend_pipe_CFLAGS=$(BOOST_CPPFLAGS) @THREADFLAGS@ $(LIBCURL_CFLAGS) $(LIBZMQ_CFLAGS) -g -O0 -I../../pdns +test_remotebackend_pipe_CXXFLAGS=$(BOOST_CPPFLAGS) @THREADFLAGS@ $(LIBCURL_CFLAGS) $(LIBZMQ_CFLAGS) -g -O0 -I../../pdns +test_remotebackend_pipe_LDADD=libtestremotebackend.la @DYNLINKFLAGS@ @THREADFLAGS@ $(BOOST_UNIT_TEST_FRAMEWORK_LDFLAGS) $(BOOST_UNIT_TEST_FRAMEWORK_LIBS) $(BOOST_SERIALIZATION_LIBS) $(BOOST_PROGRAM_OPTIONS_LIBS) @LIBDL@ $(POLARSSL_LIBS) $(LIBCURL_LIBS) $(LIBZMQ_LIBS) -test_remotebackend_unix_CFLAGS=$(BOOST_CPPFLAGS) @THREADFLAGS@ $(LIBCURL_CFLAGS) -g -O0 -I../../pdns -test_remotebackend_unix_CXXFLAGS=$(BOOST_CPPFLAGS) @THREADFLAGS@ $(LIBCURL_CFLAGS) -g -O0 -I../../pdns -test_remotebackend_unix_LDADD=libtestremotebackend.la @DYNLINKFLAGS@ @THREADFLAGS@ $(BOOST_UNIT_TEST_FRAMEWORK_LDFLAGS) $(BOOST_UNIT_TEST_FRAMEWORK_LIBS) $(BOOST_SERIALIZATION_LIBS) $(BOOST_PROGRAM_OPTIONS_LIBS) @LIBDL@ $(POLARSSL_LIBS) $(LIBCURL_LIBS) +test_remotebackend_unix_CFLAGS=$(BOOST_CPPFLAGS) @THREADFLAGS@ $(LIBCURL_CFLAGS) $(LIBZMQ_CFLAGS) -g -O0 -I../../pdns +test_remotebackend_unix_CXXFLAGS=$(BOOST_CPPFLAGS) @THREADFLAGS@ $(LIBCURL_CFLAGS) $(LIBZMQ_CFLAGS) -g -O0 -I../../pdns +test_remotebackend_unix_LDADD=libtestremotebackend.la @DYNLINKFLAGS@ @THREADFLAGS@ $(BOOST_UNIT_TEST_FRAMEWORK_LDFLAGS) $(BOOST_UNIT_TEST_FRAMEWORK_LIBS) $(BOOST_SERIALIZATION_LIBS) $(BOOST_PROGRAM_OPTIONS_LIBS) @LIBDL@ $(POLARSSL_LIBS) $(LIBCURL_LIBS) $(LIBZMQ_LIBS) -test_remotebackend_http_CFLAGS=$(BOOST_CPPFLAGS) @THREADFLAGS@ $(LIBCURL_CFLAGS) -g -O0 -I../../pdns -test_remotebackend_http_CXXFLAGS=$(BOOST_CPPFLAGS) @THREADFLAGS@ $(LIBCURL_CFLAGS) -g -O0 -I../../pdns -test_remotebackend_http_LDADD=libtestremotebackend.la @DYNLINKFLAGS@ @THREADFLAGS@ $(BOOST_UNIT_TEST_FRAMEWORK_LDFLAGS) $(BOOST_UNIT_TEST_FRAMEWORK_LIBS) $(BOOST_SERIALIZATION_LIBS) $(BOOST_PROGRAM_OPTIONS_LIBS) @LIBDL@ $(POLARSSL_LIBS) $(LIBCURL_LIBS) +test_remotebackend_http_CFLAGS=$(BOOST_CPPFLAGS) @THREADFLAGS@ $(LIBCURL_CFLAGS) $(LIBZMQ_CFLAGS) -g -O0 -I../../pdns +test_remotebackend_http_CXXFLAGS=$(BOOST_CPPFLAGS) @THREADFLAGS@ $(LIBCURL_CFLAGS) $(LIBZMQ_CFLAGS) -g -O0 -I../../pdns +test_remotebackend_http_LDADD=libtestremotebackend.la @DYNLINKFLAGS@ @THREADFLAGS@ $(BOOST_UNIT_TEST_FRAMEWORK_LDFLAGS) $(BOOST_UNIT_TEST_FRAMEWORK_LIBS) $(BOOST_SERIALIZATION_LIBS) $(BOOST_PROGRAM_OPTIONS_LIBS) @LIBDL@ $(POLARSSL_LIBS) $(LIBCURL_LIBS) $(LIBZMQ_LIBS) -test_remotebackend_post_CFLAGS=$(BOOST_CPPFLAGS) @THREADFLAGS@ $(LIBCURL_CFLAGS) -g -O0 -I../../pdns -test_remotebackend_post_CXXFLAGS=$(BOOST_CPPFLAGS) @THREADFLAGS@ $(LIBCURL_CFLAGS) -g -O0 -I../../pdns -test_remotebackend_post_LDADD=libtestremotebackend.la @DYNLINKFLAGS@ @THREADFLAGS@ $(BOOST_UNIT_TEST_FRAMEWORK_LDFLAGS) $(BOOST_UNIT_TEST_FRAMEWORK_LIBS) $(BOOST_SERIALIZATION_LIBS) $(BOOST_PROGRAM_OPTIONS_LIBS) @LIBDL@ $(POLARSSL_LIBS) $(LIBCURL_LIBS) +test_remotebackend_post_CFLAGS=$(BOOST_CPPFLAGS) @THREADFLAGS@ $(LIBCURL_CFLAGS) $(LIBZMQ_CFLAGS) -g -O0 -I../../pdns +test_remotebackend_post_CXXFLAGS=$(BOOST_CPPFLAGS) @THREADFLAGS@ $(LIBCURL_CFLAGS) $(LIBZMQ_CFLAGS) -g -O0 -I../../pdns +test_remotebackend_post_LDADD=libtestremotebackend.la @DYNLINKFLAGS@ @THREADFLAGS@ $(BOOST_UNIT_TEST_FRAMEWORK_LDFLAGS) $(BOOST_UNIT_TEST_FRAMEWORK_LIBS) $(BOOST_SERIALIZATION_LIBS) $(BOOST_PROGRAM_OPTIONS_LIBS) @LIBDL@ $(POLARSSL_LIBS) $(LIBCURL_LIBS) $(LIBZMQ_LIBS) + +test_remotebackend_json_CFLAGS=$(BOOST_CPPFLAGS) @THREADFLAGS@ $(LIBCURL_CFLAGS) $(LIBZMQ_CFLAGS) -g -O0 -I../../pdns +test_remotebackend_json_CXXFLAGS=$(BOOST_CPPFLAGS) @THREADFLAGS@ $(LIBCURL_CFLAGS) $(LIBZMQ_CFLAGS) -g -O0 -I../../pdns +test_remotebackend_json_LDADD=libtestremotebackend.la @DYNLINKFLAGS@ @THREADFLAGS@ $(BOOST_UNIT_TEST_FRAMEWORK_LDFLAGS) $(BOOST_UNIT_TEST_FRAMEWORK_LIBS) $(BOOST_SERIALIZATION_LIBS) $(BOOST_PROGRAM_OPTIONS_LIBS) @LIBDL@ $(POLARSSL_LIBS) $(LIBCURL_LIBS) $(LIBZMQ_LIBS) + +test_remotebackend_zeromq_CFLAGS=$(BOOST_CPPFLAGS) @THREADFLAGS@ $(LIBCURL_CFLAGS) $(LIBZMQ_CFLAGS) -g -O0 -I../../pdns +test_remotebackend_zeromq_CXXFLAGS=$(BOOST_CPPFLAGS) @THREADFLAGS@ $(LIBCURL_CFLAGS) $(LIBZMQ_CFLAGS) -g -O0 -I../../pdns +test_remotebackend_zeromq_LDADD=libtestremotebackend.la @DYNLINKFLAGS@ @THREADFLAGS@ $(BOOST_UNIT_TEST_FRAMEWORK_LDFLAGS) $(BOOST_UNIT_TEST_FRAMEWORK_LIBS) $(BOOST_SERIALIZATION_LIBS) $(BOOST_PROGRAM_OPTIONS_LIBS) @LIBDL@ $(POLARSSL_LIBS) $(LIBCURL_LIBS) $(LIBZMQ_LIBS) -test_remotebackend_json_CFLAGS=$(BOOST_CPPFLAGS) @THREADFLAGS@ $(LIBCURL_CFLAGS) -g -O0 -I../../pdns -test_remotebackend_json_CXXFLAGS=$(BOOST_CPPFLAGS) @THREADFLAGS@ $(LIBCURL_CFLAGS) -g -O0 -I../../pdns -test_remotebackend_json_LDADD=libtestremotebackend.la @DYNLINKFLAGS@ @THREADFLAGS@ $(BOOST_UNIT_TEST_FRAMEWORK_LDFLAGS) $(BOOST_UNIT_TEST_FRAMEWORK_LIBS) $(BOOST_SERIALIZATION_LIBS) $(BOOST_PROGRAM_OPTIONS_LIBS) @LIBDL@ $(POLARSSL_LIBS) $(LIBCURL_LIBS) diff --git a/modules/remotebackend/OBJECTFILES b/modules/remotebackend/OBJECTFILES index c125b8249..fa80d5461 100644 --- a/modules/remotebackend/OBJECTFILES +++ b/modules/remotebackend/OBJECTFILES @@ -1 +1 @@ -remotebackend.lo unixconnector.lo httpconnector.lo pipeconnector.lo +remotebackend.lo unixconnector.lo httpconnector.lo pipeconnector.lo zmqconnector.lo diff --git a/modules/remotebackend/OBJECTLIBS b/modules/remotebackend/OBJECTLIBS index 3030c7acf..68e356cc0 100644 --- a/modules/remotebackend/OBJECTLIBS +++ b/modules/remotebackend/OBJECTLIBS @@ -1 +1 @@ -$(LIBCURL_LIBS) +$(LIBCURL_LIBS) $(LIBZMQ_LIBS) diff --git a/modules/remotebackend/example.rb b/modules/remotebackend/example.rb index 3b7f2b43e..777d31dc9 100644 --- a/modules/remotebackend/example.rb +++ b/modules/remotebackend/example.rb @@ -1,6 +1,7 @@ #!/usr/bin/ruby require 'rubygems' +require 'bundler/setup' require 'json' ## this is an example stub for remote backend diff --git a/modules/remotebackend/regression-tests/Gemfile b/modules/remotebackend/regression-tests/Gemfile index 5d4c4abe2..019fc2a22 100644 --- a/modules/remotebackend/regression-tests/Gemfile +++ b/modules/remotebackend/regression-tests/Gemfile @@ -2,3 +2,4 @@ source 'https://rubygems.org' gem 'webrick' gem 'sqlite3' +gem 'zeromqrb' diff --git a/modules/remotebackend/regression-tests/Gemfile.lock b/modules/remotebackend/regression-tests/Gemfile.lock index 6c21f3dde..44b2bee99 100644 --- a/modules/remotebackend/regression-tests/Gemfile.lock +++ b/modules/remotebackend/regression-tests/Gemfile.lock @@ -1,8 +1,13 @@ GEM remote: https://rubygems.org/ specs: - sqlite3 (1.3.6) + ffi (1.9.3) + ffi-rzmq (1.0.3) + ffi + sqlite3 (1.3.8) webrick (1.3.1) + zeromqrb (0.1.1) + ffi-rzmq (~> 1.0) PLATFORMS ruby @@ -10,3 +15,4 @@ PLATFORMS DEPENDENCIES sqlite3 webrick + zeromqrb diff --git a/modules/remotebackend/regression-tests/backend.rb b/modules/remotebackend/regression-tests/backend.rb index 8f8594180..46daaadf7 100755 --- a/modules/remotebackend/regression-tests/backend.rb +++ b/modules/remotebackend/regression-tests/backend.rb @@ -1,5 +1,6 @@ -#!/usr/bin/ruby1.9.1 +#!/usr/bin/env ruby +require 'rubygems' require 'json' require 'sqlite3' diff --git a/modules/remotebackend/regression-tests/http-backend.rb b/modules/remotebackend/regression-tests/http-backend.rb index 6581a12fb..30cda13cc 100755 --- a/modules/remotebackend/regression-tests/http-backend.rb +++ b/modules/remotebackend/regression-tests/http-backend.rb @@ -1,6 +1,6 @@ -#!/usr/bin/ruby1.9.1 +#!/usr/bin/env ruby require "rubygems" -#require "bundler/setup" +require 'bundler/setup' require "webrick" require "../modules/remotebackend/regression-tests/dnsbackend" require "../modules/remotebackend/regression-tests/backend" diff --git a/modules/remotebackend/regression-tests/pipe-backend.rb b/modules/remotebackend/regression-tests/pipe-backend.rb index 27f1213d3..f9a846732 100755 --- a/modules/remotebackend/regression-tests/pipe-backend.rb +++ b/modules/remotebackend/regression-tests/pipe-backend.rb @@ -1,5 +1,6 @@ -#!/usr/bin/ruby1.9.1 - +#!/usr/bin/env ruby +require "rubygems" +require 'bundler/setup' require 'json' require '../modules/remotebackend/regression-tests/backend' diff --git a/modules/remotebackend/regression-tests/zeromq-backend.rb b/modules/remotebackend/regression-tests/zeromq-backend.rb new file mode 100755 index 000000000..6c14f772b --- /dev/null +++ b/modules/remotebackend/regression-tests/zeromq-backend.rb @@ -0,0 +1,48 @@ +#!/usr/bin/ruby1.9.1 + +#!/usr/bin/env ruby +require "rubygems" +require 'bundler/setup' +require 'json' +require 'zero_mq' +require '../modules/remotebackend/regression-tests/backend' + +h = Handler.new("../modules/remotebackend/regression-tests/remote.sqlite3") + +f = File.open "/tmp/tmp.txt","a" + +begin + context = ZeroMQ::Context.new + socket = context.socket ZMQ::REP + socket.setsockopt(ZMQ::HWM, 1000) + socket.bind("ipc:///tmp/pdns.0") + + while(true) do + line = "" + rc = socket.recv_string line + f.puts line + # expect json + input = {} + line = line.strip + next if line.empty? + begin + input = JSON.parse(line) + method = "do_#{input["method"].downcase}" + args = input["parameters"] || [] + + if h.respond_to?(method.to_sym) == false + res = false + elsif args.size > 0 + res, log = h.send(method,args) + else + res, log = h.send(method) + end + socket.send_string ({:result => res, :log => log}).to_json, 0 + f.puts({:result => res, :log => log}).to_json + rescue JSON::ParserError + socket.send_string ({:result => false, :log => "Cannot parse input #{line}"}).to_json + next + end + end +rescue SystemExit, Interrupt +end diff --git a/modules/remotebackend/remotebackend.cc b/modules/remotebackend/remotebackend.cc index 98d058210..a443b729b 100644 --- a/modules/remotebackend/remotebackend.cc +++ b/modules/remotebackend/remotebackend.cc @@ -143,6 +143,12 @@ int RemoteBackend::build() { this->connector = new HTTPConnector(options); #else throw PDNSException("Invalid connection string: http connector support not enabled. Recompile with --enable-remotebackend-http"); +#endif + } else if (type == "zeromq") { +#ifdef REMOTEBACKEND_ZEROMQ + this->connector = new ZeroMQConnector(options); +#else + throw PDNSException("Invalid connection string: zeromq connector support not enabled. Recompile with --enable-remotebackend-zeromq"); #endif } else if (type == "pipe") { this->connector = new PipeConnector(options); diff --git a/modules/remotebackend/remotebackend.hh b/modules/remotebackend/remotebackend.hh index b7bcb624d..568b94f91 100644 --- a/modules/remotebackend/remotebackend.hh +++ b/modules/remotebackend/remotebackend.hh @@ -18,7 +18,9 @@ #ifdef REMOTEBACKEND_HTTP #include #endif - +#ifdef REMOTEBACKEND_ZEROMQ +#include +#endif #define JSON_GET(obj,val,def) (obj.HasMember(val)?obj["" val ""]:def) #define JSON_ADD_MEMBER(obj, name, val, alloc) { rapidjson::Value __xval; __xval = val; obj.AddMember(name, __xval, alloc); } @@ -79,6 +81,24 @@ class HTTPConnector: public Connector { }; #endif +#ifdef REMOTEBACKEND_ZEROMQ +class ZeroMQConnector: public Connector { + public: + ZeroMQConnector(std::map options); + virtual ~ZeroMQConnector(); + virtual int send_message(const rapidjson::Document &input); + virtual int recv_message(rapidjson::Document &output); + private: + void connect(); + std::string d_endpoint; + int d_timeout; + int d_timespent; + std::map d_options; + zmq::context_t d_ctx; + zmq::socket_t d_sock; +}; +#endif + class PipeConnector: public Connector { public: diff --git a/modules/remotebackend/test-remotebackend-zeromq.cc b/modules/remotebackend/test-remotebackend-zeromq.cc new file mode 100644 index 000000000..364e97a5a --- /dev/null +++ b/modules/remotebackend/test-remotebackend-zeromq.cc @@ -0,0 +1,77 @@ +#define BOOST_TEST_DYN_LINK +#define BOOST_TEST_MAIN +#define BOOST_TEST_MODULE unit + +#include +#include +#include +#include +#include "pdns/namespaces.hh" +#include +#include +#include +#include +#include +#include +#include +#include "pdns/dnsrecords.hh" +#include +#include +#include +#include "pdns/json.hh" +#include "pdns/statbag.hh" +#include "pdns/packetcache.hh" + +StatBag S; +PacketCache PC; +ArgvMap &arg() +{ + static ArgvMap arg; + return arg; +}; + +class RemoteLoader +{ + public: + RemoteLoader(); +}; + +DNSBackend *be; + +#ifdef REMOTEBACKEND_ZEROMQ + +struct RemotebackendSetup { + RemotebackendSetup() { + be = 0; + try { + // setup minimum arguments + ::arg().set("module-dir")=""; + new RemoteLoader(); + BackendMakers().launch("remote"); + // then get us a instance of it + ::arg().set("remote-connection-string")="zeromq:endpoint=tcp://127.0.0.1:43622"; + ::arg().set("remote-dnssec")="yes"; + be = BackendMakers().all()[0]; + // load few record types to help out + SOARecordContent::report(); + NSRecordContent::report(); + ARecordContent::report(); + } catch (PDNSException &ex) { + BOOST_TEST_MESSAGE("Cannot start remotebackend: " << ex.reason ); + }; + } + ~RemotebackendSetup() { } +}; + +BOOST_GLOBAL_FIXTURE( RemotebackendSetup ); + +#else + +#include + +int main(void) { + std::cout << "No HTTP support in remotebackend - skipping test" << std::endl; + return 0; +} + +#endif diff --git a/modules/remotebackend/testrunner.sh b/modules/remotebackend/testrunner.sh index df312c95a..1cee260f6 100755 --- a/modules/remotebackend/testrunner.sh +++ b/modules/remotebackend/testrunner.sh @@ -2,6 +2,7 @@ webrick_pid="" socat_pid="" +zeromq_pid="" socat=/usr/bin/socat function start_web() { @@ -32,6 +33,31 @@ function stop_web() { fi } +function start_zeromq() { + if [ x"$REMOTEBACKEND_ZEROMQ" == "xyes" ]; then + ./unittest_zeromq.rb & + zeromq_pid=$! + # need to wait a moment + sleep 5 + fi +} + +function stop_zeromq() { + if [ ! -z "$zeromq_pid" ]; then + kill -TERM $zeromq_pid + # wait a moment for it to die + i=0 + while [ $i -lt 5 ]; do + sleep 1 + kill -0 $zeromq_pid 2>/dev/null + if [ $? -ne 0 ]; then break; fi + let i=i+1 + done + kill -0 $zeromq_pid 2>/dev/null + if [ $? -eq 0 ]; then kill -9 $zeromq_pid; fi + fi +} + function start_unix() { if [ ! -x $socat ]; then echo "Cannot find socat - cannot test (non-fatal)" @@ -92,6 +118,12 @@ case "$mode" in rv=$? stop_web ;; + test_remotebackend_zeromq) + start_zeromq + ./test_remotebackend_zeromq + rv=$? + stop_zeromq + ;; *) echo "Usage: $0 test_remotebackend_(pipe|http|post|json)" ;; diff --git a/modules/remotebackend/unittest.rb b/modules/remotebackend/unittest.rb index c44cf98dd..f169a4b3e 100644 --- a/modules/remotebackend/unittest.rb +++ b/modules/remotebackend/unittest.rb @@ -1,4 +1,5 @@ require 'rubygems' +require 'bundler/setup' require 'json' # define a simple $domain diff --git a/modules/remotebackend/unittest_http.rb b/modules/remotebackend/unittest_http.rb index e3f32daf7..3c7c34d94 100755 --- a/modules/remotebackend/unittest_http.rb +++ b/modules/remotebackend/unittest_http.rb @@ -1,10 +1,11 @@ #!/usr/bin/env ruby +require 'rubygems' +require 'bundler/setup' require 'json' require 'thread' -require "rubygems" -require "webrick" -require "./unittest" +require 'webrick' +require './unittest' class DNSBackendHandler < WEBrick::HTTPServlet::AbstractServlet def initialize(server, dnsbackend) diff --git a/modules/remotebackend/unittest_json.rb b/modules/remotebackend/unittest_json.rb index 73f14fea5..54a3e3ba5 100755 --- a/modules/remotebackend/unittest_json.rb +++ b/modules/remotebackend/unittest_json.rb @@ -1,10 +1,11 @@ #!/usr/bin/env ruby +require 'rubygems' +require 'bundler/setup' require 'json' require 'thread' -require "rubygems" -require "webrick" -require "./unittest" +require 'webrick' +require './unittest' class DNSBackendHandler < WEBrick::HTTPServlet::AbstractServlet def initialize(server, dnsbackend) diff --git a/modules/remotebackend/unittest_pipe.rb b/modules/remotebackend/unittest_pipe.rb index 35dc30705..e95652bfd 100755 --- a/modules/remotebackend/unittest_pipe.rb +++ b/modules/remotebackend/unittest_pipe.rb @@ -1,6 +1,7 @@ #!/usr/bin/env ruby require 'rubygems' +require 'bundler/setup' require 'json' require './unittest' diff --git a/modules/remotebackend/unittest_post.rb b/modules/remotebackend/unittest_post.rb index 08ac05e5e..bf9b7ed93 100755 --- a/modules/remotebackend/unittest_post.rb +++ b/modules/remotebackend/unittest_post.rb @@ -1,10 +1,11 @@ #!/usr/bin/env ruby +require 'rubygems' +require 'bundler/setup' require 'json' require 'thread' -require "rubygems" -require "webrick" -require "./unittest" +require 'webrick' +require './unittest' class DNSBackendHandler < WEBrick::HTTPServlet::AbstractServlet def initialize(server, dnsbackend) diff --git a/modules/remotebackend/unittest_zeromq.rb b/modules/remotebackend/unittest_zeromq.rb new file mode 100755 index 000000000..eb9ac0c2e --- /dev/null +++ b/modules/remotebackend/unittest_zeromq.rb @@ -0,0 +1,50 @@ +#!/usr/bin/env ruby + +require 'rubygems' +require 'bundler/setup' +require 'json' +require 'zero_mq' +require './unittest' + +h = Handler.new() +f = File.open "/tmp/tmp.txt","a" + +begin + context = ZeroMQ::Context.new + socket = context.socket ZMQ::REP + socket.setsockopt(ZMQ::HWM, 1000) + socket.bind("tcp://127.0.0.1:43622") + + print "[#{Time.now.to_s}] ZeroMQ unit test responder running\n" + + while(true) do + line = "" + rc = socket.recv_string line + f.puts line + # expect json + input = {} + line = line.strip + next if line.empty? + begin + input = JSON.parse(line) + method = "do_#{input["method"].downcase}" + args = input["parameters"] || [] + + if h.respond_to?(method.to_sym) == false + res = false + elsif args.size > 0 + res, log = h.send(method,args) + else + res, log = h.send(method) + end + socket.send_string ({:result => res, :log => log}).to_json, 0 + f.puts({:result => res, :log => log}).to_json + rescue JSON::ParserError + socket.send_string ({:result => false, :log => "Cannot parse input #{line}"}).to_json + next + end + end +rescue SystemExit, Interrupt +end + +print "[#{Time.now.to_s}] ZeroMQ unit test responder ended\n" diff --git a/modules/remotebackend/zmqconnector.cc b/modules/remotebackend/zmqconnector.cc new file mode 100644 index 000000000..3e96589ab --- /dev/null +++ b/modules/remotebackend/zmqconnector.cc @@ -0,0 +1,131 @@ +#include "remotebackend.hh" +#ifdef REMOTEBACKEND_ZEROMQ + +#include +#include +#include +#include +#include +#include "rapidjson/stringbuffer.h" +#include "rapidjson/writer.h" + +ZeroMQConnector::ZeroMQConnector(std::map options) : d_ctx(1), d_sock(d_ctx, ZMQ_REQ) { + rapidjson::Value val; + rapidjson::Document init,res; + + // lookup timeout, target and stuff + if (options.count("endpoint") == 0) { + L<d_endpoint = options.find("endpoint")->second; + this->d_options = options; + this->d_timeout=2000; + + if (options.find("timeout") != options.end()) { + this->d_timeout = boost::lexical_cast(options.find("timeout")->second); + } + + d_sock.connect(d_endpoint.c_str()); + + init.SetObject(); + val = "initialize"; + + init.AddMember("method",val, init.GetAllocator()); + val.SetObject(); + init.AddMember("parameters", val, init.GetAllocator()); + + for(std::map::iterator i = options.begin(); i != options.end(); i++) { + val = i->second.c_str(); + init["parameters"].AddMember(i->first.c_str(), val, init.GetAllocator()); + } + + this->send(init); + if (this->recv(res)==false) { + L<(message.data()), line.size()); + reinterpret_cast(message.data())[line.size()]=0; + + try { + zmq_pollitem_t item; + item.socket = d_sock; + item.events = ZMQ_POLLOUT; + // poll until it's sent or timeout is spent. try to leave + // leave few cycles for read. just in case. + for(d_timespent = 0; d_timespent < d_timeout-5; d_timespent++) { + if (zmq::poll(&item, 1, 1000)>0) { + if (d_sock.send(message, 0) == false) { + // message was not sent + L<d_endpoint << ": " << errno; + return 0; + } + return line.size(); + } + } + } catch (std::exception &ex) { + L<d_endpoint << ": " << ex.what(); + throw new PDNSException(ex.what()); + } + + return 0; +} + +int ZeroMQConnector::recv_message(rapidjson::Document &output) { + int rv = 0; + // try to receive message + zmq_pollitem_t item; + rapidjson::GenericReader , rapidjson::MemoryPoolAllocator<> > r; + zmq::message_t message; + + item.socket = d_sock; + item.events = ZMQ_POLLIN; + + try { + // do zmq::poll few times + // d_timespent should always be initialized by send_message, recv should never + // be called without send first. + for(; d_timespent < d_timeout; d_timespent++) { + if (zmq::poll(&item, 1, 1000)>0) { + // we have an event + if ((item.revents & ZMQ_POLLIN) == ZMQ_POLLIN) { + char *data; + // read something + if (d_sock.recv(&message, 0) && message.size() > 0) { + data = new char[message.size()+1]; + // convert it into json + memcpy(data, message.data(), message.size()); + data[message.size()]=0; + rapidjson::StringStream ss(data); + output.ParseStream<0>(ss); + delete [] data; + if (output.HasParseError() == false) + rv = message.size(); + else + L<d_endpoint; + break; + } else if (errno == EAGAIN) { continue; // try again } + } else { + break; + } + } + } + } + } catch (std::exception &ex) { + L<d_endpoint << ": " << ex.what(); + throw new PDNSException(ex.what()); + } + + return rv; +} + +#endif diff --git a/regression-tests/start-test-stop b/regression-tests/start-test-stop index a8bd46bbe..51e808476 100755 --- a/regression-tests/start-test-stop +++ b/regression-tests/start-test-stop @@ -306,8 +306,8 @@ gsqlite3-nodnssec gsqlite3 gsqlite3-nsec3 opendbx-sqlite3 tinydns mydns -remotebackend-pipe remotebackend-unix remotebackend-http -remotebackend-pipe-dnssec remotebackend-unix-dnssec remotebackend-http-dnssec +remotebackend-pipe remotebackend-unix remotebackend-http remotebackend-zeromq +remotebackend-pipe-dnssec remotebackend-unix-dnssec remotebackend-http-dnssec remotebackend-zeromq-dnssec #remotebackend-pipe-nsec3 remotebackend-unix-nsec3 remotebackend-http-nsec3 #remotebackend-pipe-nsec3-narrow remotebackend-unix-nsec3-narrow remotebackend-http-nsec3-narrow @@ -841,6 +841,11 @@ __EOF__ done set -e ;; + zeromq) + connstr="zeromq:endpoint=ipc:///tmp/pdns.0" + $testsdir/zeromq-backend.rb & + echo $! > pdns-remotebackend.pid + ;; unix) connstr="unix:path=/tmp/remote.socket" socat unix-listen:/tmp/remote.socket,fork exec:$testsdir/unix-backend.rb & -- 2.40.0