Commit 23d77c87 authored by Mark Andrews's avatar Mark Andrews
Browse files

4870. [test] Update included ATF library to atf-0.21 preserving

                        the ATF tool. [RT #46967]
parent af034ef1
4870. [test] Update included ATF library to atf-0.21 preserving
the ATF tool. [RT #46967]
4869. [bug] Address some cases where NULL with zero length could
be passed to memmove which is undefined behaviour and
can lead to bad optimisation. [RT #46888]
......
language: cpp
compiler:
- gcc
- clang
before_install:
- ./admin/travis-install-deps.sh
env:
- AS_ROOT=no
- AS_ROOT=yes
script:
- ./admin/travis-build.sh
notifications:
email:
- atf-log@googlegroups.com
......@@ -7,6 +7,4 @@ tp: atf-c++
tp: atf-sh
tp: test-programs
tp-glob: atf-config*
tp-glob: atf-report*
tp-glob: atf-run*
tp-glob: tools*
......@@ -7,12 +7,6 @@ include("atf-c++/Kyuafile")
include("atf-sh/Kyuafile")
include("test-programs/Kyuafile")
if fs.exists("atf-config/Kyuafile") then
include("atf-config/Kyuafile")
end
if fs.exists("atf-report/Kyuafile") then
include("atf-report/Kyuafile")
end
if fs.exists("atf-run/Kyuafile") then
include("atf-run/Kyuafile")
if fs.exists("tools/Kyuafile") then
include("tools/Kyuafile")
end
#
# Automated Testing Framework (atf)
#
# Copyright (c) 2007 The NetBSD Foundation, Inc.
# All rights reserved.
#
......@@ -25,11 +22,11 @@
# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
# IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
atf_aclocal_DATA =
BUILT_SOURCES =
CLEANFILES =
DIST_HOOKS =
EXTRA_DIST =
bin_PROGRAMS =
dist_man_MANS =
......@@ -54,10 +51,7 @@ include doc/Makefile.am.inc
include test-programs/Makefile.am.inc
if ENABLE_TOOLS
include atf-report/Makefile.am.inc
include atf-config/Makefile.am.inc
include atf-run/Makefile.am.inc
include atf-version/Makefile.am.inc
include tools/Makefile.am.inc
endif
#
......@@ -75,6 +69,10 @@ EXTRA_DIST += $(doc_DATA) INSTALL README
TESTS_ENVIRONMENT = PATH=$(prefix)/bin:$${PATH} \
PKG_CONFIG_PATH=$(prefix)/lib/pkgconfig
# Allow the caller to override the configuration file to passed to our
# test runs below.
KYUA_TEST_CONFIG_FILE = none
testsdir = $(exec_prefix)/tests
pkgtestsdir = $(testsdir)/$(PACKAGE)
......@@ -105,7 +103,8 @@ if !ENABLE_TOOLS
INSTALLCHECK_TARGETS += installcheck-kyua
endif
installcheck-kyua:
cd $(pkgtestsdir) && $(TESTS_ENVIRONMENT) $(KYUA) test
cd $(pkgtestsdir) && $(TESTS_ENVIRONMENT) \
$(KYUA) --config='$(KYUA_TEST_CONFIG_FILE)' test
endif
installcheck-local: $(INSTALLCHECK_TARGETS)
......@@ -117,16 +116,20 @@ endif
EXTRA_DIST += $(pkgtests_DATA)
BUILD_SH_TP = \
echo "Creating $${dst}"; \
echo "\#! $(bindir)/atf-sh" >$${dst}; \
cat $${src} >>$${dst}; \
chmod +x $${dst}
test -d "$$(dirname "$${dst}")" || mkdir -p "$$(dirname "$${dst}")"; \
echo "\#! $(bindir)/atf-sh" >"$${dst}"; \
if [ -n "$${substs}" ]; then \
cat $${src} | sed $${substs} >>"$${dst}"; \
else \
cat $${src} >>"$${dst}"; \
fi; \
chmod +x "$${dst}"
#
# Custom targets.
#
dist-hook: forbid-dist
DIST_HOOKS += forbid-dist
if ENABLE_TOOLS
forbid-dist:
@true
......@@ -141,14 +144,16 @@ PHONY_TARGETS += clean-all
clean-all:
GIT="$(GIT)" $(SH) $(srcdir)/admin/clean-all.sh
PHONY_TARGETS += release
release:
$(SH) $(srcdir)/admin/release.sh $(PACKAGE_VERSION) $(DIST_ARCHIVES)
.PHONY: $(PHONY_TARGETS)
PHONY_TARGETS += release-test
release-test:
$(SH) $(srcdir)/admin/release-test.sh $(DIST_ARCHIVES)
# TODO(jmmv): Remove after atf 0.22.
install-data-hook:
cd $(DESTDIR)$(man3dir) && \
for binding in c c++ sh; do \
rm -f "atf-$${binding}-api.3"; \
$(LN_S) "atf-$${binding}.3" "atf-$${binding}-api.3"; \
done
.PHONY: $(PHONY_TARGETS)
dist-hook: $(DIST_HOOKS)
# vim: syntax=make:noexpandtab:shiftwidth=8:softtabstop=8
This diff is collapsed.
Major changes between releases Automated Testing Framework
===========================================================================
Changes in version 0.21
***********************
Released on October 23rd, 2014.
* Restored the atf(7) manual page to serve as a reference to all the other
manual pages shipped by ATF.
* Added the -s flag to atf-sh to support specifying the shell interpreter
to be used.
* Removed ATF_WORKDIR. The only remaining consumers have been converted to
use the standard TMPDIR environment variable. As a benefit, and because
Kyua forces the TMPDIR to live within the test case's work directory,
any stale files left behind by ATF will be automatically cleaned up.
* Documented the environment variables recognized by each component in the
relevant manual pages. This information was lost with the atf-config(1)
removal.
* Added a new "require.diskspace" metadata property to test cases so that
they can specify the minimum amount of disk space required for the test
to run.
* Renamed the atf-{c,c++,sh}-api(3) manual pages to atf-{c,c++,sh}(3) for
discoverability purposes. Symbolic links are provided for the time
being to still make the old names visible.
* Issue #5: Recommend the (expected, actual) idiom for calls to the test
macros in the manual pages.
* Issue #7: Stopped catching unhandled exceptions in atf-c++ tests. This
propagates the crash to the caller, which in turn allows it to obtain
proper debugging information. In particular, Kyua should now be able to
extract a stacktrace pinpointing the problem.
* Issue #8: Fixed atf-c/macros_test:use test failures spotted by the clang
that ships with FreeBSD 11.0-CURRENT.
* Issue #12: Improved documentation of atf-sh(3) and atf-check(1) by better
explaining how they relate to each other.
* Issue #14: Stopped setting 'set -e' in atf-sh. This setting was
initially added as a way to enable a "strict" mode in the library and to
make test cases fail fast when they run unprotected commands. However,
doing so in the library is surprising as the responsibility of enabling
'set -e' should be on the user's code. Also, 'set -e' introduces
inconsistent behavior on subshells and users do not expect that.
* Issue #15: Fixed atf_utils_{fork,wait} to support nested calls.
* Issue #16: Fixed test failures (by removing a long-standing hack) on
systems that lack \e support in printf(1).
* Issue #19: Removed stale references to atf-config and atf-run.
Changes in version 0.19
***********************
Experimental version released on February 7th, 2014.
This is the last release to bundle the code for the deprecated tools.
The next release will drop their code and will stop worrying about
backwards compatibility between the ATF libraries and what the old tools
may or may not support.
If you still require the old tools for some reason, grab a copy of the
'tools' directory now. The code in this directory is standalone and
does not depend on any internal details of atf-c++ any longer.
* Various fixes and improvements to support running as part of the FreeBSD
test suite.
* Project hosting moved from Google Code (as a subproject of Kyua) to
GitHub (as a first-class project). The main reason for the change is
the suppression of binary downloads in Google Code on Jan 15th, 2014.
See https://github.com/jmmv/atf/
* Removed builtin help from atf-sh(1) and atf-check(1) for simplicity
reasons. In other words, their -h option is gone.
* Moved the code of the deprecated tools into a 'tools' directory and
completely decoupled their code from the internals of atf-c++. The
reason for this is to painlessly allow a third-party to maintain a
copy of these tools after we delete them because upcoming changes to
atf-c++ would break the stale tools.
Changes in version 0.18
***********************
Experimental version released on November 16th, 2013.
* Issue 45: Added require.memory support in atf-run for FreeBSD.
* Fixed an issue with the handling of cin with libc++.
* Issue 64: Fixed various mandoc formatting warnings.
* NetBSD PR bin/48284: Made atf-check flush its progress message to
stdout so that an interrupted test case always shows the last message
being executed.
* NetBSD PR bin/48285: Fixed atf_check examples in atf-sh-api(3).
Changes in version 0.17
***********************
......
# Welcome to the ATF project!
ATF, or Automated Testing Framework, is a **collection of libraries** to
write test programs in **C, C++ and POSIX shell**.
The ATF libraries offer a simple API. The API is orthogonal through the
various bindings, allowing developers to quickly learn how to write test
programs in different languages.
ATF-based test programs offer a **consistent end-user command-line
interface** to allow both humans and automation to run the tests.
ATF-based test programs **rely on an execution engine** to be run and
this execution engine is *not* shipped with ATF.
**[Kyua](https://github.com/jmmv/kyua/) is the engine of choice.**
## Download
Formal releases for source files are available for download from GitHub:
* [atf 0.20](../../releases/tag/atf-0.20), released on February 7th, 2014.
## Installation
You are encouraged to install binary packages for your operating system
wherever available:
* Fedora 20 and above: install the `atf` package with `yum install atf`.
* FreeBSD 10.0 and above: install the `atf` package with `pkg install atf`.
* NetBSD with pkgsrc: install the `pkgsrc/devel/atf` package.
Should you want to build and install ATF from the source tree provided
here, follow the instructions in the [INSTALL file](INSTALL).
## Support
Please use the
[atf-discuss mailing list](https://groups.google.com/forum/#!forum/atf-discuss)
for any support inquiries related to `atf-c`, `atf-c++` or `atf-sh`.
If you have any questions on Kyua proper, please use the
[kyua-discuss mailing list](https://groups.google.com/forum/#!forum/kyua-discuss)
instead.
Things to do Automated Testing Framework
===========================================================================
Last revised: November 30th, 2010
This document includes the list of things that need to be done in ATF that
are most requested by the users. This information used to be available in
an ad-hoc bug tracker but that proved to be a bad idea. I have collected
all worthy comments in here.
Please note that most work these days is going into Kyua (see
http://code.google.com/p/kyua/). The ideas listed here apply to the
components of ATF that have *not* been migrated to the new codebase yet.
For bug reports or ideas that apply to the components that already have
been migrated, please use the bug tracker in the URL above. Similarly,
whenever a component is migrated, the ideas in this file should be revised
and migrated to the new bug tracker where appropriate.
---------------------------------------------------------------------------
Add build-time checks to atf-sh
The 0.7 release introduced build-time tests to atf-c and atf-c++, but not
to atf-sh. Expose the functionality to the shell interface.
This will probably require writing an atf-build utility that exposes the C
code and can be called from the shell.
---------------------------------------------------------------------------
Revisit what to do when an Atffile lists a non-existent file
---------------------------------------------------------------------------
Add ATF_CHECK* versions to atf-c++ to support non-fatal tests
---------------------------------------------------------------------------
Implement race-condition tests
gcooper:
I would think that stress/negative tests would be of more value than race
condition tests (they're similar, but not exactly the same in my mind).
In particular,
1. Feed through as much data as possible to determine where reporting
breaks down.
2. Feed through data quickly and terminate ASAP. The data should be
captured. Terminate child applications with unexpected exit codes and
signals (in particular, SIGCHLD, SIGPIPE, exit codes that terminate,
etc).
3. Open up a file descriptor in the test application, don't close the file
descriptor.
4. fork(2) a process; don't wait(2) for the application to complete.
There are other scenarios that could be exercised, but these are the ones
I could think of off the topic of my head.
--
jmmv:
1. The thing is: how do you express any of this in a portable/abstract
interface? How do you express that a test case "receives data"? What
does that exactly mean? I don't think the framework should care about
this: each test should be free to decide where its data is and how to
deal with it.
2. Ditto.
3. Not sure I understand your request, but testing for "unexpected exit
codes" is already supported. See wiki:DesignXFail for the feature
design details.
4. What's the problem with this case? The test case exits right away after
terminating the execution of its body; any open file descriptors,
leaked memory, etc. die with it.
5. forking and not waiting for a subprocess was a problem already
addressed.
I kinda have an idea of what Antti means with "race condition tests", but
every time I have tried to describe my understanding of matters I seem to
be wrong. Would be nice to have a clear description of what this involves;
in particular, what are the expectations from the framework and how should
the feature be exposed.
As of now, what I understand by "race condition test" is: a test case that
exercises a race condition. The test case may finish without triggering
the race, in which case it just exists with a successful status.
Otherwise, if the race triggers, the test case gets stuck and times out.
The result should be reported as an "expected failure" different from
timeout.
--
pooka:
Yup. Plus some atf-wide mechanism for the operator to supply some kind of
guideline if the test should try to trigger the race for a second or for
an hour.
--
jmmv:
Alright. While mocking up some code for this, I think that your two
requests are complementary.
On the one hand, when you are talking about a "race condition" test you
really mean an "expected race condition" test. Correct? If so, we need to
extend the xfail mechanism to add one more case, which is to report any
failures as a race condition error and, if there is no failure, report the
test as successful.
On the other hand, the atf-wide mechanism to support how long the test
should run for can be thought as a "stress test" mechanism. I.e. run this
test for X time / iterations and report its results regularly without
involving xfail at all.
So, with this in mind:
* For a test that triggers an unfixed race condition, you set xfail to
race mode and define the test as a stress test. Any failures are
reported as expected failures.
* For a test that verifies a supposedly-fixed race condition, you do *not*
set xfail to race mode, and only set the test to stress test. Any
failures are reported as real failures.
These stress test cases implement a single iteration of the test and
atf-run is in charge of running the test several times, stopping on the
first failure.
Does that make sense?
---------------------------------------------------------------------------
Implement ATF_REQUIRE_ERRNO
pooka:
Most of the lines in tests against system functionality are:
if (syscall(args) == -1)
atf_tc_fail_errno("flop")
Some shorthand would be helpful, like ATF_REQUIRE_ERRNO(syscall(args))
Also, a variant which allows arbitrary return value checks (e.g. "!= 0" or
"< 124" or "!= size") would be nice.
--
gcooper:
There's a problem with this request; not all functions fail in the same
way ... in particular compare the pthread family of functions (which
return errno) vs many native syscalls. Furthermore, compare some
fcntl-like syscalls vs other syscalls. One size fits all solutions may not
be a wise idea in this case, so I think that the problem statement needs
to be better defined, because the above request is too loose.
FWIW, there's also a TEST macro in LTP, which tests for non-zero status,
and sets an appropriate set of global variables for errnos and return
codes, respectively. It was a good idea, but has been mostly abandoned
because it's too difficult to define a success and failure in a universal
manner, so I think that we need to be careful with what's implemented in
ATF to not repeat the mistakes that others have made.
--
jmmv:
I think you've got a good point.
This was mostly intended to simplify the handling of the stupid errno
global variable. I think this is valuable to have, but maybe the
macro/function name should be different because _ERRNO can be confusing.
Probably something like an ATF_CHECK_LIBC / ATF_CHECK_PTHREAD approach
would be more flexible and simple.
===========================================================================
vim: filetype=text:textwidth=75:expandtab:shiftwidth=2:softtabstop=2
This diff is collapsed.
#
# Automated Testing Framework (atf)
#
# Copyright (c) 2007 The NetBSD Foundation, Inc.
# All rights reserved.
#
......@@ -25,7 +22,6 @@
# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
# IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
dist-hook: check-style
PHONY_TARGETS += check-style
......
#! /bin/sh
# Wrapper for Microsoft lib.exe
me=ar-lib
scriptversion=2012-03-01.08; # UTC
# Copyright (C) 2010-2017 Free Software Foundation, Inc.
# Written by Peter Rosin <peda@lysator.liu.se>.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# As a special exception to the GNU General Public License, if you
# distribute this file as part of a program that contains a
# configuration script generated by Autoconf, you may include it under
# the same distribution terms that you use for the rest of that program.
# This file is maintained in Automake, please report
# bugs to <bug-automake@gnu.org> or send patches to
# <automake-patches@gnu.org>.
# func_error message
func_error ()
{
echo "$me: $1" 1>&2
exit 1
}
file_conv=
# func_file_conv build_file
# Convert a $build file to $host form and store it in $file
# Currently only supports Windows hosts.
func_file_conv ()
{
file=$1
case $file in
/ | /[!/]*) # absolute file, and not a UNC file
if test -z "$file_conv"; then
# lazily determine how to convert abs files
case `uname -s` in
MINGW*)
file_conv=mingw
;;
CYGWIN*)
file_conv=cygwin
;;
*)
file_conv=wine
;;
esac
fi
case $file_conv in
mingw)
file=`cmd //C echo "$file " | sed -e 's/"\(.*\) " *$/\1/'`
;;
cygwin)
file=`cygpath -m "$file" || echo "$file"`
;;
wine)
file=`winepath -w "$file" || echo "$file"`
;;
esac
;;
esac
}
# func_at_file at_file operation archive
# Iterate over all members in AT_FILE performing OPERATION on ARCHIVE
# for each of them.
# When interpreting the content of the @FILE, do NOT use func_file_conv,
# since the user would need to supply preconverted file names to
# binutils ar, at least for MinGW.
func_at_file ()
{
operation=$2
archive=$3
at_file_contents=`cat "$1"`
eval set x "$at_file_contents"
shift
for member
do
$AR -NOLOGO $operation:"$member" "$archive" || exit $?
done
}
case $1 in
'')
func_error "no command. Try '$0 --help' for more information."
;;
-h | --h*)
cat <<EOF
Usage: $me [--help] [--version] PROGRAM ACTION ARCHIVE [MEMBER...]
Members may be specified in a file named with @FILE.
EOF
exit $?
;;
-v | --v*)
echo "$me, version $scriptversion"
exit $?
;;
esac
if test $# -lt 3; then
func_error "you must specify a program, an action and an archive"
fi
AR=$1
shift
while :
do
if test $# -lt 2; then
func_error "you must specify a program, an action and an archive"
fi
case $1 in