From d3c09b9b1307e022883801000ae36bcb5eef71e8 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Mon, 3 Dec 2018 09:27:35 +0900 Subject: [PATCH] Add PGXS options to control TAP and isolation tests, take two MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit The following options are added for extensions: - TAP_TESTS, to allow an extention to run TAP tests which are the ones present in t/*.pl. A subset of tests can always be run with the existing PROVE_TESTS for developers. - ISOLATION, to define a list of isolation tests. - ISOLATION_OPTS, to pass custom options to isolation_tester. A couple of custom Makefile rules have been accumulated across the tree to cover the lack of facility in PGXS for a couple of releases when using those test suites, which are all now replaced with the new flags, without reducing the test coverage. Note that tests of contrib/bloom/ are not enabled yet, as those are proving unstable in the buildfarm. Author: Michael Paquier Reviewed-by: Adam Berlin, Álvaro Herrera, Tom Lane, Nikolay Shaplov, Arthur Zakirov Discussion: https://postgr.es/m/20180906014849.GG2726@paquier.xyz --- contrib/bloom/Makefile | 7 ++- contrib/oid2name/Makefile | 10 +--- contrib/test_decoding/Makefile | 65 +++++----------------- contrib/vacuumlo/Makefile | 10 +--- doc/src/sgml/extend.sgml | 61 +++++++++++++++++++- src/makefiles/pgxs.mk | 32 ++++++++++- src/test/modules/brin/.gitignore | 2 +- src/test/modules/brin/Makefile | 25 +-------- src/test/modules/commit_ts/Makefile | 7 +-- src/test/modules/snapshot_too_old/Makefile | 35 +++--------- src/test/modules/test_pg_dump/Makefile | 6 +- 11 files changed, 127 insertions(+), 133 deletions(-) diff --git a/contrib/bloom/Makefile b/contrib/bloom/Makefile index 13bd397b70..146878870e 100644 --- a/contrib/bloom/Makefile +++ b/contrib/bloom/Makefile @@ -9,6 +9,10 @@ PGFILEDESC = "bloom access method - signature file based index" REGRESS = bloom +# Disable TAP tests for this module for now, as these are unstable on several +# buildfarm environments. +# TAP_TESTS = 1 + ifdef USE_PGXS PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) @@ -19,6 +23,3 @@ top_builddir = ../.. include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk endif - -wal-check: temp-install - $(prove_check) diff --git a/contrib/oid2name/Makefile b/contrib/oid2name/Makefile index 908e078714..361a80a7a1 100644 --- a/contrib/oid2name/Makefile +++ b/contrib/oid2name/Makefile @@ -6,11 +6,11 @@ PGAPPICON = win32 PROGRAM = oid2name OBJS = oid2name.o $(WIN32RES) +TAP_TESTS = 1 + PG_CPPFLAGS = -I$(libpq_srcdir) PG_LIBS_INTERNAL = $(libpq_pgport) -EXTRA_CLEAN = tmp_check - ifdef USE_PGXS PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) @@ -21,9 +21,3 @@ top_builddir = ../.. include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk endif - -check: - $(prove_check) - -installcheck: - $(prove_installcheck) diff --git a/contrib/test_decoding/Makefile b/contrib/test_decoding/Makefile index afcab930f7..4afb1d963e 100644 --- a/contrib/test_decoding/Makefile +++ b/contrib/test_decoding/Makefile @@ -3,9 +3,18 @@ MODULES = test_decoding PGFILEDESC = "test_decoding - example of a logical decoding output plugin" -# Note: because we don't tell the Makefile there are any regression tests, -# we have to clean those result files explicitly -EXTRA_CLEAN = $(pg_regress_clean_files) +REGRESS = ddl xact rewrite toast permissions decoding_in_xact \ + decoding_into_rel binary prepared replorigin time messages \ + spill slot truncate +ISOLATION = mxact delayed_startup ondisk_startup concurrent_ddl_dml \ + oldest_xmin snapshot_transfer + +REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/test_decoding/logical.conf +ISOLATION_OPTS = --temp-config $(top_srcdir)/contrib/test_decoding/logical.conf + +# Disabled because these tests require "wal_level=logical", which +# typical installcheck users do not have (e.g. buildfarm clients). +NO_INSTALLCHECK = 1 ifdef USE_PGXS PG_CONFIG = pg_config @@ -18,52 +27,8 @@ include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk endif -# Disabled because these tests require "wal_level=logical", which -# typical installcheck users do not have (e.g. buildfarm clients). -installcheck:; - # But it can nonetheless be very helpful to run tests on preexisting # installation, allow to do so, but only if requested explicitly. -installcheck-force: regresscheck-install-force isolationcheck-install-force - -check: regresscheck isolationcheck - -submake-regress: - $(MAKE) -C $(top_builddir)/src/test/regress all - -submake-isolation: - $(MAKE) -C $(top_builddir)/src/test/isolation all - -submake-test_decoding: - $(MAKE) -C $(top_builddir)/contrib/test_decoding - -REGRESSCHECKS=ddl xact rewrite toast permissions decoding_in_xact \ - decoding_into_rel binary prepared replorigin time messages \ - spill slot truncate - -regresscheck: | submake-regress submake-test_decoding temp-install - $(pg_regress_check) \ - --temp-config $(top_srcdir)/contrib/test_decoding/logical.conf \ - $(REGRESSCHECKS) - -regresscheck-install-force: | submake-regress submake-test_decoding temp-install - $(pg_regress_installcheck) \ - $(REGRESSCHECKS) - -ISOLATIONCHECKS=mxact delayed_startup ondisk_startup concurrent_ddl_dml \ - oldest_xmin snapshot_transfer - -isolationcheck: | submake-isolation submake-test_decoding temp-install - $(pg_isolation_regress_check) \ - --temp-config $(top_srcdir)/contrib/test_decoding/logical.conf \ - $(ISOLATIONCHECKS) - -isolationcheck-install-force: all | submake-isolation submake-test_decoding temp-install - $(pg_isolation_regress_installcheck) \ - $(ISOLATIONCHECKS) - -.PHONY: submake-test_decoding submake-regress check \ - regresscheck regresscheck-install-force \ - isolationcheck isolationcheck-install-force - -temp-install: EXTRA_INSTALL=contrib/test_decoding +installcheck-force: + $(pg_regress_installcheck) $(REGRESS) + $(pg_isolation_regress_installcheck) $(ISOLATION) diff --git a/contrib/vacuumlo/Makefile b/contrib/vacuumlo/Makefile index 5de506151e..3efcb46735 100644 --- a/contrib/vacuumlo/Makefile +++ b/contrib/vacuumlo/Makefile @@ -6,11 +6,11 @@ PGAPPICON = win32 PROGRAM = vacuumlo OBJS = vacuumlo.o $(WIN32RES) +TAP_TESTS = 1 + PG_CPPFLAGS = -I$(libpq_srcdir) PG_LIBS_INTERNAL = $(libpq_pgport) -EXTRA_CLEAN = tmp_check - ifdef USE_PGXS PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) @@ -21,9 +21,3 @@ top_builddir = ../.. include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk endif - -check: - $(prove_check) - -installcheck: - $(prove_installcheck) diff --git a/doc/src/sgml/extend.sgml b/doc/src/sgml/extend.sgml index 695e07fb38..a6b77c1cfe 100644 --- a/doc/src/sgml/extend.sgml +++ b/doc/src/sgml/extend.sgml @@ -1303,6 +1303,34 @@ include $(PGXS) + + ISOLATION + + + list of isolation test cases, see below for more details + + + + + + ISOLATION_OPTS + + + additional switches to pass to + pg_isolation_regress + + + + + + TAP_TESTS + + + switch defining if TAP tests need to be run, see below + + + + NO_INSTALLCHECK @@ -1423,13 +1451,42 @@ make VPATH=/path/to/extension/source/tree install have all expected files. + + The scripts listed in the ISOLATION variable are used + for tests stressing behavior of concurrent session with your module, which + can be invoked by make installcheck after doing + make install. For this to work you must have a + running PostgreSQL server. The script files + listed in ISOLATION must appear in a subdirectory + named specs/ in your extension's directory. These files + must have extension .spec, which must not be included + in the ISOLATION list in the makefile. For each test + there should also be a file containing the expected output in a + subdirectory named expected/, with the same stem and + extension .out. make installcheck + executes each test script, and compares the resulting output to the + matching expected file. Any differences will be written to the file + output_iso/regression.diffs in + diff -c format. Note that trying to run a test that is + missing its expected file will be reported as trouble, so + make sure you have all expected files. + + + + TAP_TESTS enables the use of TAP tests. Data from each + run is present in a subdirectory named tmp_check/. + See also for more details. + + The easiest way to create the expected files is to create empty files, then do a test run (which will of course report differences). Inspect the actual result files found in the results/ - directory, then copy them to expected/ if they match - what you expect from the test. + directory (for tests in REGRESS), or + output_iso/results/ directory (for tests in + ISOLATION), then copy them to + expected/ if they match what you expect from the test. diff --git a/src/makefiles/pgxs.mk b/src/makefiles/pgxs.mk index 070d151018..7e7c54f022 100644 --- a/src/makefiles/pgxs.mk +++ b/src/makefiles/pgxs.mk @@ -46,6 +46,9 @@ # HEADERS_built_$(MODULE) -- as above but built first (also NOT cleaned) # REGRESS -- list of regression test cases (without suffix) # REGRESS_OPTS -- additional switches to pass to pg_regress +# TAP_TESTS -- switch to enable TAP tests +# ISOLATION -- list of isolation test cases +# ISOLATION_OPTS -- additional switches to pass to pg_isolation_regress # NO_INSTALLCHECK -- don't define an installcheck target, useful e.g. if # tests require special configuration, or don't use pg_regress # EXTRA_CLEAN -- extra files to remove in 'make clean' @@ -349,6 +352,12 @@ ifeq ($(PORTNAME), win) rm -f regress.def endif endif # REGRESS +ifdef TAP_TESTS + rm -rf tmp_check/ +endif +ifdef ISOLATION + rm -rf output_iso/ tmp_check_iso/ +endif ifdef MODULE_big clean: clean-lib @@ -383,28 +392,47 @@ $(test_files_build): $(abs_builddir)/%: $(srcdir)/% $(MKDIR_P) $(dir $@) ln -s $< $@ endif # VPATH +endif # REGRESS .PHONY: submake submake: ifndef PGXS $(MAKE) -C $(top_builddir)/src/test/regress pg_regress$(X) + $(MAKE) -C $(top_builddir)/src/test/isolation all endif -# against installed postmaster +# Standard rules to run regression tests including multiple test suites. +# Runs against an installed postmaster. ifndef NO_INSTALLCHECK installcheck: submake $(REGRESS_PREP) +ifdef REGRESS $(pg_regress_installcheck) $(REGRESS_OPTS) $(REGRESS) endif +ifdef ISOLATION + $(pg_isolation_regress_installcheck) $(ISOLATION_OPTS) $(ISOLATION) +endif +ifdef TAP_TESTS + $(prove_installcheck) +endif +endif # NO_INSTALLCHECK +# Runs independently of any installation ifdef PGXS check: @echo '"$(MAKE) check" is not supported.' @echo 'Do "$(MAKE) install", then "$(MAKE) installcheck" instead.' else check: submake $(REGRESS_PREP) +ifdef REGRESS $(pg_regress_check) $(REGRESS_OPTS) $(REGRESS) endif -endif # REGRESS +ifdef ISOLATION + $(pg_isolation_regress_check) $(ISOLATION_OPTS) $(ISOLATION) +endif +ifdef TAP_TESTS + $(prove_check) +endif +endif # PGXS ifndef NO_TEMP_INSTALL temp-install: EXTRA_INSTALL+=$(subdir) diff --git a/src/test/modules/brin/.gitignore b/src/test/modules/brin/.gitignore index 62bbe8f6b1..44f600cb6c 100644 --- a/src/test/modules/brin/.gitignore +++ b/src/test/modules/brin/.gitignore @@ -1,3 +1,3 @@ # Generated subdirectories -/isolation_output/ +/output_iso/ /tmp_check/ diff --git a/src/test/modules/brin/Makefile b/src/test/modules/brin/Makefile index 566655cd61..c871593906 100644 --- a/src/test/modules/brin/Makefile +++ b/src/test/modules/brin/Makefile @@ -1,12 +1,9 @@ # src/test/modules/brin/Makefile -# Note: because we don't tell the Makefile there are any regression tests, -# we have to clean those result files explicitly -EXTRA_CLEAN = $(pg_regress_clean_files) ./isolation_output +EXTRA_INSTALL = contrib/pageinspect -EXTRA_INSTALL=contrib/pageinspect - -ISOLATIONCHECKS=summarization-and-inprogress-insertion +ISOLATION = summarization-and-inprogress-insertion +TAP_TESTS = 1 ifdef USE_PGXS PG_CONFIG = pg_config @@ -18,19 +15,3 @@ top_builddir = ../../../.. include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk endif - -check: isolation-check prove-check - -isolation-check: | submake-isolation temp-install - $(MKDIR_P) isolation_output - $(pg_isolation_regress_check) \ - --outputdir=./isolation_output \ - $(ISOLATIONCHECKS) - -prove-check: | temp-install - $(prove_check) - -.PHONY: check isolation-check prove-check - -submake-isolation: - $(MAKE) -C $(top_builddir)/src/test/isolation all diff --git a/src/test/modules/commit_ts/Makefile b/src/test/modules/commit_ts/Makefile index 6a9c3971fb..113bcfa210 100644 --- a/src/test/modules/commit_ts/Makefile +++ b/src/test/modules/commit_ts/Makefile @@ -6,6 +6,8 @@ REGRESS_OPTS = --temp-config=$(top_srcdir)/src/test/modules/commit_ts/commit_ts. # which typical installcheck users do not have (e.g. buildfarm clients). NO_INSTALLCHECK = 1 +TAP_TESTS = 1 + ifdef USE_PGXS PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) @@ -16,8 +18,3 @@ top_builddir = ../../../.. include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk endif - -check: prove-check - -prove-check: | temp-install - $(prove_check) diff --git a/src/test/modules/snapshot_too_old/Makefile b/src/test/modules/snapshot_too_old/Makefile index b6d998f320..dfb4537f63 100644 --- a/src/test/modules/snapshot_too_old/Makefile +++ b/src/test/modules/snapshot_too_old/Makefile @@ -4,7 +4,12 @@ # we have to clean those result files explicitly EXTRA_CLEAN = $(pg_regress_clean_files) -ISOLATIONCHECKS=sto_using_cursor sto_using_select sto_using_hash_index +ISOLATION = sto_using_cursor sto_using_select sto_using_hash_index +ISOLATION_OPTS = --temp-config $(top_srcdir)/src/test/modules/snapshot_too_old/sto.conf + +# Disabled because these tests require "old_snapshot_threshold" >= 0, which +# typical installcheck users do not have (e.g. buildfarm clients). +NO_INSTALLCHECK = 1 ifdef USE_PGXS PG_CONFIG = pg_config @@ -17,31 +22,7 @@ include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk endif -# Disabled because these tests require "old_snapshot_threshold" >= 0, which -# typical installcheck users do not have (e.g. buildfarm clients). -installcheck:; - # But it can nonetheless be very helpful to run tests on preexisting # installation, allow to do so, but only if requested explicitly. -installcheck-force: isolationcheck-install-force - -check: isolationcheck - -submake-isolation: - $(MAKE) -C $(top_builddir)/src/test/isolation all - -submake-test_snapshot_too_old: - $(MAKE) -C $(top_builddir)/src/test/modules/snapshot_too_old - -isolationcheck: | submake-isolation submake-test_snapshot_too_old temp-install - $(pg_isolation_regress_check) \ - --temp-config $(top_srcdir)/src/test/modules/snapshot_too_old/sto.conf \ - $(ISOLATIONCHECKS) - -isolationcheck-install-force: all | submake-isolation submake-test_snapshot_too_old temp-install - $(pg_isolation_regress_installcheck) \ - $(ISOLATIONCHECKS) - -.PHONY: check submake-test_snapshot_too_old isolationcheck isolationcheck-install-force - -temp-install: EXTRA_INSTALL=src/test/modules/snapshot_too_old +installcheck-force: + $(pg_isolation_regress_installcheck) $(ISOLATION) diff --git a/src/test/modules/test_pg_dump/Makefile b/src/test/modules/test_pg_dump/Makefile index c64b353707..6123b994f6 100644 --- a/src/test/modules/test_pg_dump/Makefile +++ b/src/test/modules/test_pg_dump/Makefile @@ -7,6 +7,7 @@ EXTENSION = test_pg_dump DATA = test_pg_dump--1.0.sql REGRESS = test_pg_dump +TAP_TESTS = 1 ifdef USE_PGXS PG_CONFIG = pg_config @@ -18,8 +19,3 @@ top_builddir = ../../../.. include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk endif - -check: prove-check - -prove-check: | temp-install - $(prove_check) -- 2.40.0